File substitutes

.gitignore

@@ -124,6 +124,7 @@ Makefile.in
 /tests/common.sh
 /tests/dummy
 /tests/result*
+/tests/file-substituter.sh
 
 # /tests/lang/
 /tests/lang/*.out

src/libexpr/Makefile.am

@@ -8,7 +8,7 @@ libexpr_la_SOURCES = \
 pkginclude_HEADERS = \
  nixexpr.hh eval.hh eval-inline.hh lexer-tab.hh parser-tab.hh \
  get-drvs.hh attr-path.hh value-to-xml.hh common-opts.hh \
- names.hh symbol-table.hh value.hh
+ names.hh symbol-table.hh context.hh value.hh
 
 libexpr_la_LIBADD = ../libutil/libutil.la ../libstore/libstore.la \
  ../boost/format/libformat.la @BDW_GC_LIBS@

src/libexpr/context.hh

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <cstring>
+
+namespace nix {
+
+
+struct ContextEntry
+{
+    const char *path;
+    const char *output;
+    bool discardOutputs;
+};
+
+
+struct CompareContextEntry
+{
+    inline bool operator() (const ContextEntry* const & c1, const ContextEntry* const & c2) {
+        return (strcmp(c1->path,c2->path) < 0) || ((strcmp(c1->path,c2->path) == 0) && (strcmp(c1->output,c2->output) < 0));
+    }
+};
+
+
+typedef std::set<const ContextEntry *, CompareContextEntry> ContextEntrySet;
+
+
+}

src/libexpr/eval.cc

@@ -142,7 +142,7 @@ EvalState::EvalState()
     , baseEnvDispl(0)
     , staticBaseEnv(false, 0)
 {
-    nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0;
+    nrEnvs = nrValuesInEnvs = nrValues = nrListElems = nrContextEntries = nrContextEntryBytes = 0;
     nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0;
     nrListConcats = nrPrimOpCalls = nrFunctionCalls = 0;
     countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
@@ -275,15 +275,15 @@ void mkString(Value & v, const char * s)
 }
 
 
-void mkString(Value & v, const string & s, const PathSet & context)
+void mkString(Value & v, const string & s, const ContextEntrySet & context)
 {
     mkString(v, s.c_str());
     if (!context.empty()) {
         unsigned int n = 0;
-        v.string.context = (const char * *)
-            GC_MALLOC((context.size() + 1) * sizeof(char *));
-        foreach (PathSet::const_iterator, i, context)  
-            v.string.context[n++] = GC_STRDUP(i->c_str());
+        v.string.context = (const ContextEntry * *)
+            GC_MALLOC((context.size() + 1) * sizeof(ContextEntry *));
+        foreach (ContextEntrySet::const_iterator, i, context)  
+            v.string.context[n++] = *i;
         v.string.context[n] = 0;
     }
 }
@@ -322,6 +322,20 @@ Value * EvalState::allocValue()
 }
 
 
+ContextEntry * EvalState::allocContextEntry(const Path & path, const string & output) {
+    nrContextEntries++;
+    nrContextEntryBytes += sizeof(ContextEntry) + (path.size() + 1) + (output.size() + 1);
+    ContextEntry * c = (ContextEntry *) GC_MALLOC(sizeof(ContextEntry) + (path.size() + 1) + (output.size() + 1));
+    c->path = ((const char *) c) + sizeof(ContextEntry);
+    c->output = c->path + (path.size() + 1);
+    // !!! Commented out because of zero-out behaviour of GC_MALLOC, can we rely on false == 0?
+    // c->discardOutputs = false;
+    memcpy((void *) c->path, path.c_str(), path.size() + 1);
+    memcpy((void *) c->output, output.c_str(), output.size() + 1);
+    return c;
+}
+
+
 Env & EvalState::allocEnv(unsigned int size)
 {
     nrEnvs++;
@@ -441,6 +455,24 @@ void EvalState::evalFile(const Path & path, Value & v)
 }
 
 
+void EvalState::evalSubstitutableFile(const Path & path, Value & v)
+{
+    FileEvalCache::iterator i = fileEvalCache.find(path);
+    if (i == fileEvalCache.end()) {
+        startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path);
+        Expr * e = parseExprFromSubstitutableFile(path);
+        try {
+            eval(e, v);
+        } catch (Error & e) {
+            addErrorPrefix(e, "while evaluating the file `%1%':\n", path);
+            throw;
+        }
+        fileEvalCache[path] = v;
+    } else
+        v = i->second;
+}
+
+
 void EvalState::eval(Expr * e, Value & v)
 {
     e->eval(*this, baseEnv, v);
@@ -948,7 +980,7 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists)
 
 void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 {
-    PathSet context;
+    ContextEntrySet context;
     std::ostringstream s;
 
     bool first = true, isPath = false;
@@ -1030,15 +1062,15 @@ string EvalState::forceString(Value & v)
 }
 
 
-void copyContext(const Value & v, PathSet & context)
+void copyContext(const Value & v, ContextEntrySet & context)
 {
     if (v.string.context)
-        for (const char * * p = v.string.context; *p; ++p) 
+        for (const ContextEntry * * p = v.string.context; *p; ++p)
             context.insert(*p);
 }
 
 
-string EvalState::forceString(Value & v, PathSet & context)
+string EvalState::forceString(Value & v, ContextEntrySet & context)
 {
     string s = forceString(v);
     copyContext(v, context);
@@ -1051,7 +1083,7 @@ string EvalState::forceStringNoCtx(Value & v)
     string s = forceString(v);
     if (v.string.context)
         throwEvalError("the string `%1%' is not allowed to refer to a store path (such as `%2%')",
-            v.string.s, v.string.context[0]);
+            v.string.s, v.string.context[0]->path);
     return s;
 }
 
@@ -1067,7 +1099,7 @@ bool EvalState::isDerivation(Value & v)
 }
 
 
-string EvalState::coerceToString(Value & v, PathSet & context,
+string EvalState::coerceToString(Value & v, ContextEntrySet & context,
     bool coerceMore, bool copyToStore)
 {
     forceValue(v);
@@ -1091,15 +1123,18 @@ string EvalState::coerceToString(Value & v, PathSet & context,
         if (srcToStore[path] != "")
             dstPath = srcToStore[path];
         else {
-            dstPath = settings.readOnlyMode
-                ? computeStorePathForPath(path).first
-                : store->addToStore(path);
+            if (settings.readOnlyMode && isInStore(path) && !store->isValidPath(toStorePath(path)))
+                dstPath = computeStorePathForSubstitutablePath(path).first;
+            else
+                dstPath = settings.readOnlyMode
+                    ? computeStorePathForPath(path).first
+                    : store->addToStore(path);
             srcToStore[path] = dstPath;
             printMsg(lvlChatty, format("copied source `%1%' -> `%2%'")
                 % path % dstPath);
         }
 
-        context.insert(dstPath);
+        context.insert(allocContextEntry(dstPath));
         return dstPath;
     }
 
@@ -1137,7 +1172,7 @@ string EvalState::coerceToString(Value & v, PathSet & context,
 }
 
 
-Path EvalState::coerceToPath(Value & v, PathSet & context)
+Path EvalState::coerceToPath(Value & v, ContextEntrySet & context)
 {
     string path = coerceToString(v, context, false, false);
     if (path == "" || path[0] != '/')
@@ -1170,11 +1205,13 @@ bool EvalState::eqValues(Value & v1, Value & v2)
         case tString: {
             /* Compare both the string and its context. */
             if (strcmp(v1.string.s, v2.string.s) != 0) return false;
-            const char * * p = v1.string.context, * * q = v2.string.context;
+            const ContextEntry * * p = v1.string.context, * * q = v2.string.context;
             if (!p && !q) return true;
             if (!p || !q) return false;
-            for ( ; *p && *q; ++p, ++q)
-                if (strcmp(*p, *q) != 0) return false;
+            for ( ; *p && *q; ++p, ++q) {
+                if (strcmp((*p)->path, (*q)->path) != 0) return false;
+                if (strcmp((*p)->output, (*q)->output) != 0) return false;
+	    }
             if (*p || *q) return false;
             return true;
         }
@@ -1243,6 +1280,8 @@ void EvalState::printStats()
     printMsg(v, format("  list concatenations: %1%") % nrListConcats);
     printMsg(v, format("  values allocated: %1% (%2% bytes)")
         % nrValues % (nrValues * sizeof(Value)));
+    printMsg(v, format("  string context entries allocated: %1% (%2% bytes)")
+        % nrContextEntries % nrContextEntryBytes);
     printMsg(v, format("  attribute sets allocated: %1%") % nrAttrsets);
     printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
     printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);

src/libexpr/eval.hh

@@ -71,10 +71,9 @@ struct Attr
     }
 };
 
+void mkString(Value & v, const string & s, const ContextEntrySet & context = ContextEntrySet());
 
-void mkString(Value & v, const string & s, const PathSet & context = PathSet());
-
-void copyContext(const Value & v, PathSet & context);
+void copyContext(const Value & v, ContextEntrySet & context);
 
 
 /* Cache for calls to addToStore(); maps source paths to the store
@@ -124,13 +123,21 @@ public:
        refers to a directory, then "/default.nix" is appended. */
     Expr * parseExprFromFile(Path path);
 
+    /* Parse a Nix expression from the substitute for the specified file.
+       If `path' refers to a directory, then "/default.nix" is appended. */
+    Expr * parseExprFromSubstitutableFile(Path path);
+
     /* Parse a Nix expression from the specified string. */
     Expr * parseExprFromString(const string & s, const Path & basePath);
 
     /* Evaluate an expression read from the given file to normal
        form. */
     void evalFile(const Path & path, Value & v);
 
+    /* Evaluate an expression read from the substutite for the
+       given file to normal form. */
+    void evalSubstitutableFile(const Path & path, Value & v);
+
     /* Look up a file in the search path. */
     Path findFile(const string & path);
 
@@ -160,7 +167,7 @@ public:
     inline void forceList(Value & v);
     void forceFunction(Value & v); // either lambda or primop
     string forceString(Value & v);
-    string forceString(Value & v, PathSet & context);
+    string forceString(Value & v, ContextEntrySet & context);
     string forceStringNoCtx(Value & v);
 
     /* Return true iff the value `v' denotes a derivation (i.e. a
@@ -171,13 +178,13 @@ public:
        string.  If `coerceMore' is set, also converts nulls, integers,
        booleans and lists to a string.  If `copyToStore' is set,
        referenced paths are copied to the Nix store as a side effect.q */
-    string coerceToString(Value & v, PathSet & context,
+    string coerceToString(Value & v, ContextEntrySet & context,
         bool coerceMore = false, bool copyToStore = true);
 
     /* Path coercion.  Converts strings, paths and derivations to a
        path.  The result is guaranteed to be a canonicalised, absolute
        path.  Nothing is copied to the store. */
-    Path coerceToPath(Value & v, PathSet & context);
+    Path coerceToPath(Value & v, ContextEntrySet & context);
 
 private:
 
@@ -224,6 +231,9 @@ public:
 
     /* Allocation primitives. */
     Value * allocValue();
+
+    ContextEntry * allocContextEntry(const Path & path, const string & output = string());
+
     Env & allocEnv(unsigned int size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
@@ -242,6 +252,8 @@ private:
     unsigned long nrEnvs;
     unsigned long nrValuesInEnvs;
     unsigned long nrValues;
+    unsigned long nrContextEntries;
+    unsigned long nrContextEntryBytes;
     unsigned long nrListElems;
     unsigned long nrAttrsets;
     unsigned long nrOpUpdates;