diff --git a/.gitignore b/.gitignore index dd8abf2d225..1ff093cde77 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,7 @@ Makefile.in /tests/common.sh /tests/dummy /tests/result* +/tests/file-substituter.sh # /tests/lang/ /tests/lang/*.out diff --git a/src/libexpr/Makefile.am b/src/libexpr/Makefile.am index 3e7e7e856a5..c7be4ec96ce 100644 --- a/src/libexpr/Makefile.am +++ b/src/libexpr/Makefile.am @@ -8,7 +8,7 @@ libexpr_la_SOURCES = \ pkginclude_HEADERS = \ nixexpr.hh eval.hh eval-inline.hh lexer-tab.hh parser-tab.hh \ get-drvs.hh attr-path.hh value-to-xml.hh common-opts.hh \ - names.hh symbol-table.hh value.hh + names.hh symbol-table.hh context.hh value.hh libexpr_la_LIBADD = ../libutil/libutil.la ../libstore/libstore.la \ ../boost/format/libformat.la @BDW_GC_LIBS@ diff --git a/src/libexpr/context.hh b/src/libexpr/context.hh new file mode 100644 index 00000000000..83720854f11 --- /dev/null +++ b/src/libexpr/context.hh @@ -0,0 +1,27 @@ +#pragma once + +#include + +namespace nix { + + +struct ContextEntry +{ + const char *path; + const char *output; + bool discardOutputs; +}; + + +struct CompareContextEntry +{ + inline bool operator() (const ContextEntry* const & c1, const ContextEntry* const & c2) { + return (strcmp(c1->path,c2->path) < 0) || ((strcmp(c1->path,c2->path) == 0) && (strcmp(c1->output,c2->output) < 0)); + } +}; + + +typedef std::set ContextEntrySet; + + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c10177223e5..7d79e004c1f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -142,7 +142,7 @@ EvalState::EvalState() , baseEnvDispl(0) , staticBaseEnv(false, 0) { - nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0; + nrEnvs = nrValuesInEnvs = nrValues = nrListElems = nrContextEntries = nrContextEntryBytes = 0; nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0; nrListConcats = nrPrimOpCalls = nrFunctionCalls = 0; countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; @@ -275,15 +275,15 @@ void mkString(Value & v, const char * s) } -void mkString(Value & v, const string & s, const PathSet & context) +void mkString(Value & v, const string & s, const ContextEntrySet & context) { mkString(v, s.c_str()); if (!context.empty()) { unsigned int n = 0; - v.string.context = (const char * *) - GC_MALLOC((context.size() + 1) * sizeof(char *)); - foreach (PathSet::const_iterator, i, context) - v.string.context[n++] = GC_STRDUP(i->c_str()); + v.string.context = (const ContextEntry * *) + GC_MALLOC((context.size() + 1) * sizeof(ContextEntry *)); + foreach (ContextEntrySet::const_iterator, i, context) + v.string.context[n++] = *i; v.string.context[n] = 0; } } @@ -322,6 +322,20 @@ Value * EvalState::allocValue() } +ContextEntry * EvalState::allocContextEntry(const Path & path, const string & output) { + nrContextEntries++; + nrContextEntryBytes += sizeof(ContextEntry) + (path.size() + 1) + (output.size() + 1); + ContextEntry * c = (ContextEntry *) GC_MALLOC(sizeof(ContextEntry) + (path.size() + 1) + (output.size() + 1)); + c->path = ((const char *) c) + sizeof(ContextEntry); + c->output = c->path + (path.size() + 1); + // !!! Commented out because of zero-out behaviour of GC_MALLOC, can we rely on false == 0? + // c->discardOutputs = false; + memcpy((void *) c->path, path.c_str(), path.size() + 1); + memcpy((void *) c->output, output.c_str(), output.size() + 1); + return c; +} + + Env & EvalState::allocEnv(unsigned int size) { nrEnvs++; @@ -441,6 +455,24 @@ void EvalState::evalFile(const Path & path, Value & v) } +void EvalState::evalSubstitutableFile(const Path & path, Value & v) +{ + FileEvalCache::iterator i = fileEvalCache.find(path); + if (i == fileEvalCache.end()) { + startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path); + Expr * e = parseExprFromSubstitutableFile(path); + try { + eval(e, v); + } catch (Error & e) { + addErrorPrefix(e, "while evaluating the file `%1%':\n", path); + throw; + } + fileEvalCache[path] = v; + } else + v = i->second; +} + + void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); @@ -948,7 +980,7 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists) void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) { - PathSet context; + ContextEntrySet context; std::ostringstream s; bool first = true, isPath = false; @@ -1030,15 +1062,15 @@ string EvalState::forceString(Value & v) } -void copyContext(const Value & v, PathSet & context) +void copyContext(const Value & v, ContextEntrySet & context) { if (v.string.context) - for (const char * * p = v.string.context; *p; ++p) + for (const ContextEntry * * p = v.string.context; *p; ++p) context.insert(*p); } -string EvalState::forceString(Value & v, PathSet & context) +string EvalState::forceString(Value & v, ContextEntrySet & context) { string s = forceString(v); copyContext(v, context); @@ -1051,7 +1083,7 @@ string EvalState::forceStringNoCtx(Value & v) string s = forceString(v); if (v.string.context) throwEvalError("the string `%1%' is not allowed to refer to a store path (such as `%2%')", - v.string.s, v.string.context[0]); + v.string.s, v.string.context[0]->path); return s; } @@ -1067,7 +1099,7 @@ bool EvalState::isDerivation(Value & v) } -string EvalState::coerceToString(Value & v, PathSet & context, +string EvalState::coerceToString(Value & v, ContextEntrySet & context, bool coerceMore, bool copyToStore) { forceValue(v); @@ -1091,15 +1123,18 @@ string EvalState::coerceToString(Value & v, PathSet & context, if (srcToStore[path] != "") dstPath = srcToStore[path]; else { - dstPath = settings.readOnlyMode - ? computeStorePathForPath(path).first - : store->addToStore(path); + if (settings.readOnlyMode && isInStore(path) && !store->isValidPath(toStorePath(path))) + dstPath = computeStorePathForSubstitutablePath(path).first; + else + dstPath = settings.readOnlyMode + ? computeStorePathForPath(path).first + : store->addToStore(path); srcToStore[path] = dstPath; printMsg(lvlChatty, format("copied source `%1%' -> `%2%'") % path % dstPath); } - context.insert(dstPath); + context.insert(allocContextEntry(dstPath)); return dstPath; } @@ -1137,7 +1172,7 @@ string EvalState::coerceToString(Value & v, PathSet & context, } -Path EvalState::coerceToPath(Value & v, PathSet & context) +Path EvalState::coerceToPath(Value & v, ContextEntrySet & context) { string path = coerceToString(v, context, false, false); if (path == "" || path[0] != '/') @@ -1170,11 +1205,13 @@ bool EvalState::eqValues(Value & v1, Value & v2) case tString: { /* Compare both the string and its context. */ if (strcmp(v1.string.s, v2.string.s) != 0) return false; - const char * * p = v1.string.context, * * q = v2.string.context; + const ContextEntry * * p = v1.string.context, * * q = v2.string.context; if (!p && !q) return true; if (!p || !q) return false; - for ( ; *p && *q; ++p, ++q) - if (strcmp(*p, *q) != 0) return false; + for ( ; *p && *q; ++p, ++q) { + if (strcmp((*p)->path, (*q)->path) != 0) return false; + if (strcmp((*p)->output, (*q)->output) != 0) return false; + } if (*p || *q) return false; return true; } @@ -1243,6 +1280,8 @@ void EvalState::printStats() printMsg(v, format(" list concatenations: %1%") % nrListConcats); printMsg(v, format(" values allocated: %1% (%2% bytes)") % nrValues % (nrValues * sizeof(Value))); + printMsg(v, format(" string context entries allocated: %1% (%2% bytes)") + % nrContextEntries % nrContextEntryBytes); printMsg(v, format(" attribute sets allocated: %1%") % nrAttrsets); printMsg(v, format(" right-biased unions: %1%") % nrOpUpdates); printMsg(v, format(" values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a1f26a0566d..ac85a4c6c5e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -71,10 +71,9 @@ struct Attr } }; +void mkString(Value & v, const string & s, const ContextEntrySet & context = ContextEntrySet()); -void mkString(Value & v, const string & s, const PathSet & context = PathSet()); - -void copyContext(const Value & v, PathSet & context); +void copyContext(const Value & v, ContextEntrySet & context); /* Cache for calls to addToStore(); maps source paths to the store @@ -124,6 +123,10 @@ public: refers to a directory, then "/default.nix" is appended. */ Expr * parseExprFromFile(Path path); + /* Parse a Nix expression from the substitute for the specified file. + If `path' refers to a directory, then "/default.nix" is appended. */ + Expr * parseExprFromSubstitutableFile(Path path); + /* Parse a Nix expression from the specified string. */ Expr * parseExprFromString(const string & s, const Path & basePath); @@ -131,6 +134,10 @@ public: form. */ void evalFile(const Path & path, Value & v); + /* Evaluate an expression read from the substutite for the + given file to normal form. */ + void evalSubstitutableFile(const Path & path, Value & v); + /* Look up a file in the search path. */ Path findFile(const string & path); @@ -160,7 +167,7 @@ public: inline void forceList(Value & v); void forceFunction(Value & v); // either lambda or primop string forceString(Value & v); - string forceString(Value & v, PathSet & context); + string forceString(Value & v, ContextEntrySet & context); string forceStringNoCtx(Value & v); /* Return true iff the value `v' denotes a derivation (i.e. a @@ -171,13 +178,13 @@ public: string. If `coerceMore' is set, also converts nulls, integers, booleans and lists to a string. If `copyToStore' is set, referenced paths are copied to the Nix store as a side effect.q */ - string coerceToString(Value & v, PathSet & context, + string coerceToString(Value & v, ContextEntrySet & context, bool coerceMore = false, bool copyToStore = true); /* Path coercion. Converts strings, paths and derivations to a path. The result is guaranteed to be a canonicalised, absolute path. Nothing is copied to the store. */ - Path coerceToPath(Value & v, PathSet & context); + Path coerceToPath(Value & v, ContextEntrySet & context); private: @@ -224,6 +231,9 @@ public: /* Allocation primitives. */ Value * allocValue(); + + ContextEntry * allocContextEntry(const Path & path, const string & output = string()); + Env & allocEnv(unsigned int size); Value * allocAttr(Value & vAttrs, const Symbol & name); @@ -242,6 +252,8 @@ private: unsigned long nrEnvs; unsigned long nrValuesInEnvs; unsigned long nrValues; + unsigned long nrContextEntries; + unsigned long nrContextEntryBytes; unsigned long nrListElems; unsigned long nrAttrsets; unsigned long nrOpUpdates; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 6670d0636a1..7ef022e2829 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -6,11 +6,34 @@ namespace nix { +string DrvInfo::queryName(EvalState & state) const +{ + if (name == "" && attrs) { + Bindings::iterator i = attrs->find(state.sName); + (string &) name = state.forceStringNoCtx(*i->value); + } + return name; +} + + +string DrvInfo::querySystem(EvalState & state) const +{ + if (system == "" && attrs) { + Bindings::iterator i = attrs->find(state.sSystem); + if (i == attrs->end()) + (string &) system = "unknown"; + else + (string &) system = state.forceStringNoCtx(*i->value); + } + return system; +} + + string DrvInfo::queryDrvPath(EvalState & state) const { if (drvPath == "" && attrs) { Bindings::iterator i = attrs->find(state.sDrvPath); - PathSet context; + ContextEntrySet context; (string &) drvPath = i != attrs->end() ? state.coerceToPath(*i->value, context) : ""; } return drvPath; @@ -21,18 +44,41 @@ string DrvInfo::queryOutPath(EvalState & state) const { if (outPath == "" && attrs) { Bindings::iterator i = attrs->find(state.sOutPath); - PathSet context; + ContextEntrySet context; (string &) outPath = i != attrs->end() ? state.coerceToPath(*i->value, context) : ""; } return outPath; } +static MetaValue createMetaValue(EvalState & state, Value & v) +{ + MetaValue value; + try { + state.forceValue(v); + if (v.type == tString) { + value.type = MetaValue::tpString; + value.stringValue = v.string.s; + } else if (v.type == tInt) { + value.type = MetaValue::tpInt; + value.intValue = v.integer; + } else if (v.type == tList) { + value.type = MetaValue::tpStrings; + for (unsigned int j = 0; j < v.list.length; ++j) + value.stringValues.push_back(state.forceStringNoCtx(*v.list.elems[j])); + } else + value.type = MetaValue::tpNone; + } catch (ImportReadOnlyError & e) { + value.type = MetaValue::tpNone; + } + return value; +} MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const { if (metaInfoRead) return meta; (bool &) metaInfoRead = true; + ((MetaInfo &) meta).clear(); Bindings::iterator a = attrs->find(state.sMeta); if (a == attrs->end()) return meta; /* fine, empty meta information */ @@ -40,20 +86,8 @@ MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const state.forceAttrs(*a->value); foreach (Bindings::iterator, i, *a->value->attrs) { - MetaValue value; - state.forceValue(*i->value); - if (i->value->type == tString) { - value.type = MetaValue::tpString; - value.stringValue = i->value->string.s; - } else if (i->value->type == tInt) { - value.type = MetaValue::tpInt; - value.intValue = i->value->integer; - } else if (i->value->type == tList) { - value.type = MetaValue::tpStrings; - for (unsigned int j = 0; j < i->value->list.length; ++j) - value.stringValues.push_back(state.forceStringNoCtx(*i->value->list.elems[j])); - } else continue; - ((MetaInfo &) meta)[i->name] = value; + MetaValue value = createMetaValue(state, *i->value); + if (value.type != MetaValue::tpNone) ((MetaInfo &) meta)[i->name] = value; } return meta; @@ -62,8 +96,19 @@ MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const MetaValue DrvInfo::queryMetaInfo(EvalState & state, const string & name) const { - /* !!! evaluates all meta attributes => inefficient */ - return queryMetaInfo(state)[name]; + if (metaInfoRead) return ((MetaInfo &)meta)[name]; + if (meta.count(name)) return ((MetaInfo &)meta)[name]; + + Bindings::iterator m = attrs->find(state.sMeta); + if (m == attrs->end()) return ((MetaInfo &)meta)[name]; + + state.forceAttrs(*m->value); + Bindings::iterator i = m->value->attrs->find(state.symbols.create(name)); + if (i == m->value->attrs->end()) return ((MetaInfo &)meta)[name]; + MetaValue value = createMetaValue(state, *i->value); + if (value.type != MetaValue::tpNone) ((MetaInfo &) meta)[name] = value; + + return ((MetaInfo &)meta)[name]; } @@ -100,13 +145,6 @@ static bool getDerivation(EvalState & state, Value & v, Bindings::iterator i = v.attrs->find(state.sName); /* !!! We really would like to have a decent back trace here. */ if (i == v.attrs->end()) throw TypeError("derivation name missing"); - drv.name = state.forceStringNoCtx(*i->value); - - Bindings::iterator i2 = v.attrs->find(state.sSystem); - if (i2 == v.attrs->end()) - drv.system = "unknown"; - else - drv.system = state.forceStringNoCtx(*i2->value); drv.attrs = v.attrs; @@ -117,6 +155,8 @@ static bool getDerivation(EvalState & state, Value & v, } catch (AssertionError & e) { return false; + } catch (ImportReadOnlyError & e) { + return false; } } diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 25d8baa559b..b0176d958ee 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -26,6 +26,8 @@ typedef std::map MetaInfo; struct DrvInfo { private: + string name; + string system; string drvPath; string outPath; @@ -35,20 +37,30 @@ private: bool failed; // set if we get an AssertionError public: - string name; string attrPath; /* path towards the derivation */ - string system; /* !!! make this private */ Bindings * attrs; DrvInfo() : metaInfoRead(false), failed(false), attrs(0) { }; + string queryName(EvalState & state) const; + string querySystem(EvalState & state) const; string queryDrvPath(EvalState & state) const; string queryOutPath(EvalState & state) const; MetaInfo queryMetaInfo(EvalState & state) const; MetaValue queryMetaInfo(EvalState & state, const string & name) const; + void setName(const string & s) + { + name = s; + } + + void setSystem(const string & s) + { + system = s; + } + void setDrvPath(const string & s) { drvPath = s; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index bc6c3287c79..0eac3d69d88 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -16,6 +16,7 @@ MakeError(ThrownError, AssertionError) MakeError(Abort, EvalError) MakeError(TypeError, EvalError) MakeError(ImportError, EvalError) // error building an imported derivation +MakeError(ImportReadOnlyError, EvalError) // error when trying to import a derivation in read-only mode /* Position objects. */ diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 1819da5e1ca..25023b08bc1 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -464,6 +464,7 @@ formal #include #include +#include namespace nix { @@ -524,6 +525,57 @@ Expr * EvalState::parseExprFromFile(Path path) } +Expr * EvalState::parseExprFromSubstitutableFile(Path path) +{ + assert(path[0] == '/'); + + SubstitutableFileInfos infos; + store->querySubstitutableFileInfos(singleton(path), infos); + if (!infos.count(path)) + throw ImportReadOnlyError(format( + "cannot parse `%1%' because its parent store path is invalid and it has no file substitute") + % path); + SubstitutableFileInfo info = infos[path]; + + /* If `path' is a symlink, follow it. This is so that relative + path references work. */ + if (info.type == tpSymlink) { + path = absPath(info.target, dirOf(path)); + if (!isInStore(path) || store->isValidPath(toStorePath(path))) + return parseExprFromFile(path); + else + return parseExprFromSubstitutableFile(path); + } + + /* If `path' refers to a directory, append `/default.nix'. */ + if (info.type == tpDirectory) + return parseExprFromSubstitutableFile(path + "/default.nix"); + + /* Read and parse the input file, unless it's already in the parse + tree cache. */ + Expr * e = parseTrees[path]; + if (!e) { + if (!store->querySubstitutableFiles(singleton(path)).count(path)) + throw ImportReadOnlyError(format( + "cannot parse `%1%' because its parent store path is invalid and it has no file substitute") + % path); + FdPair fds; + store->readSubstitutableFile(path, fds); + unsigned char * buf = new unsigned char[info.regular.length]; + AutoDeleteArray d(buf); + try { + readFull(fds.first, buf, info.regular.length); + e = parse(string((char *) buf, info.regular.length).c_str(), path, dirOf(path)); + } catch (EndOfFile err) { + throw Error(format("substituter failed: %1%") % chomp(drainFD(fds.second))); + } + parseTrees[path] = e; + } + + return e; +} + + Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) { return parse(s.c_str(), "(string)", basePath); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d3809e6984a..138951f922f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -24,59 +24,56 @@ namespace nix { *************************************************************/ -/* Decode a context string ‘!!’ into a pair . */ -std::pair decodeContext(const string & s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return std::pair(string(s, index + 1), string(s, 1, index - 1)); - } else - return std::pair(s, ""); -} - - /* Load and evaluate an expression from path specified by the argument. */ static void prim_import(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; Path path = state.coerceToPath(*args[0], context); - foreach (PathSet::iterator, i, context) { - Path ctx = decodeContext(*i).first; - assert(isStorePath(ctx)); - if (!store->isValidPath(ctx)) - throw EvalError(format("cannot import `%1%', since path `%2%' is not valid") - % path % ctx); - if (isDerivation(ctx)) - try { - /* For performance, prefetch all substitute info. */ - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - queryMissing(*store, singleton(ctx), - willBuild, willSubstitute, unknown, downloadSize, narSize); - - /* !!! If using a substitute, we only need to fetch - the selected output of this derivation. */ - store->buildPaths(singleton(ctx)); - } catch (Error & e) { - throw ImportError(e.msg()); + if (!settings.readOnlyMode) { + foreach (ContextEntrySet::iterator, i, context) { + Path ctx = (*i)->path; + string outputName = (*i)->output; + assert(isStorePath(ctx)); + if (!store->isValidPath(ctx)) + throw EvalError(format("cannot import `%1%', since path `%2%' is not valid") + % path % ctx); + if (isDerivation(ctx) && !(*i)->discardOutputs) { + Derivation drv = derivationFromPath(*store, ctx); + + if (outputName.empty() || !store->isValidPath(drv.outputs[outputName].path)) { + try { + /* For performance, prefetch all substitute info. */ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + queryMissing(*store, singleton(ctx), + willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!outputName.empty() && willSubstitute.count(drv.outputs[outputName].path)) + store->ensurePath(drv.outputs[outputName].path); + else + store->buildPaths(singleton(ctx)); + } catch (Error & e) { + throw ImportError(e.msg()); + } + } } + } } if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { Derivation drv = parseDerivation(readFile(path)); Value & w = *state.allocValue(); state.mkAttrs(w, 1 + drv.outputs.size()); - mkString(*state.allocAttr(w, state.sDrvPath), path, singleton("=" + path)); + mkString(*state.allocAttr(w, state.sDrvPath), path, singleton(state.allocContextEntry(path))); state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size()); unsigned int outputs_index = 0; Value * outputsVal = w.attrs->find(state.symbols.create("outputs"))->value; foreach (DerivationOutputs::iterator, i, drv.outputs) { mkString(*state.allocAttr(w, state.symbols.create(i->first)), - i->second.path, singleton("!" + i->first + "!" + path)); + i->second.path, singleton(state.allocContextEntry(path,i->first))); mkString(*(outputsVal->list.elems[outputs_index++] = state.allocValue()), i->first); } @@ -88,7 +85,10 @@ static void prim_import(EvalState & state, Value * * args, Value & v) mkApp(v, fun, w); state.forceAttrs(v); } else { - state.evalFile(path, v); + if (settings.readOnlyMode && isInStore(path) && !store->isValidPath(toStorePath(path))) + state.evalSubstitutableFile(path, v); + else + state.evalFile(path, v); } } @@ -220,7 +220,7 @@ static void prim_genericClosure(EvalState & state, Value * * args, Value & v) static void prim_abort(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; throw Abort(format("evaluation aborted with the following error message: `%1%'") % state.coerceToString(*args[0], context)); } @@ -228,7 +228,7 @@ static void prim_abort(EvalState & state, Value * * args, Value & v) static void prim_throw(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; throw ThrownError(format("user-thrown exception: %1%") % state.coerceToString(*args[0], context)); } @@ -240,7 +240,7 @@ static void prim_addErrorContext(EvalState & state, Value * * args, Value & v) state.forceValue(*args[1]); v = *args[1]; } catch (Error & e) { - PathSet context; + ContextEntrySet context; e.addPrefix(format("%1%\n") % state.coerceToString(*args[0], context)); throw; } @@ -320,7 +320,7 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v) /* Build the derivation expression by processing the attributes. */ Derivation drv; - PathSet context; + ContextEntrySet context; string outputHash, outputHashAlgo; bool outputHashRecursive = false; @@ -394,46 +394,24 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v) /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - foreach (PathSet::iterator, i, context) { - Path path = *i; - - /* Paths marked with `=' denote that the path of a derivation - is explicitly passed to the builder. Since that allows the - builder to gain access to every path in the dependency - graph of the derivation (including all outputs), all paths - in the graph must be added to this derivation's list of - inputs to ensure that they are available when the builder - runs. */ - if (path.at(0) == '=') { + foreach (ContextEntrySet::iterator, i, context) { + Path path = (*i)->path; + string output = (*i)->output; + if (!isDerivation(path) || (*i)->discardOutputs) + drv.inputSrcs.insert(path); + else if (!output.empty()) + drv.inputDrvs[path].insert(output); + else { /* !!! This doesn't work if readOnlyMode is set. */ - PathSet refs; computeFSClosure(*store, string(path, 1), refs); + PathSet refs; computeFSClosure(*store, path, refs); foreach (PathSet::iterator, j, refs) { drv.inputSrcs.insert(*j); if (isDerivation(*j)) drv.inputDrvs[*j] = store->queryDerivationOutputNames(*j); } } - - /* See prim_unsafeDiscardOutputDependency. */ - else if (path.at(0) == '~') - drv.inputSrcs.insert(string(path, 1)); - - /* Handle derivation outputs of the form ‘!!’. */ - else if (path.at(0) == '!') { - std::pair ctx = decodeContext(path); - drv.inputDrvs[ctx.first].insert(ctx.second); - } - - /* Handle derivation contexts returned by - ‘builtins.storePath’. */ - else if (isDerivation(path)) - drv.inputDrvs[path] = store->queryDerivationOutputNames(path); - - /* Otherwise it's a source file. */ - else - drv.inputSrcs.insert(path); } - + /* Do we have all required attributes? */ if (drv.builder == "") throw EvalError("required attribute `builder' missing"); @@ -498,11 +476,10 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v) drvHashes[drvPath] = hashDerivationModulo(*store, drv); state.mkAttrs(v, 1 + drv.outputs.size()); - mkString(*state.allocAttr(v, state.sDrvPath), drvPath, singleton("=" + drvPath)); - foreach (DerivationOutputs::iterator, i, drv.outputs) { + mkString(*state.allocAttr(v, state.sDrvPath), drvPath, singleton(state.allocContextEntry(drvPath))); + foreach (DerivationOutputs::iterator, i, drv.outputs) mkString(*state.allocAttr(v, state.symbols.create(i->first)), - i->second.path, singleton("!" + i->first + "!" + drvPath)); - } + i->second.path, singleton(state.allocContextEntry(drvPath, i->first))); v.attrs->sort(); } @@ -515,7 +492,7 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v) /* Convert the argument to a path. !!! obsolete? */ static void prim_toPath(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; Path path = state.coerceToPath(*args[0], context); mkString(v, canonPath(path), context); } @@ -531,7 +508,7 @@ static void prim_toPath(EvalState & state, Value * * args, Value & v) corner cases. */ static void prim_storePath(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; Path path = state.coerceToPath(*args[0], context); /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so @@ -542,14 +519,14 @@ static void prim_storePath(EvalState & state, Value * * args, Value & v) Path path2 = toStorePath(path); if (!store->isValidPath(path2)) throw EvalError(format("store path `%1%' is not valid") % path2); - context.insert(path2); + context.insert(state.allocContextEntry(path2)); mkString(v, path, context); } static void prim_pathExists(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; Path path = state.coerceToPath(*args[0], context); if (!context.empty()) throw EvalError(format("string `%1%' cannot refer to other paths") % path); @@ -561,7 +538,7 @@ static void prim_pathExists(EvalState & state, Value * * args, Value & v) following the last slash. */ static void prim_baseNameOf(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; mkString(v, baseNameOf(state.coerceToString(*args[0], context)), context); } @@ -571,7 +548,7 @@ static void prim_baseNameOf(EvalState & state, Value * * args, Value & v) of the argument. */ static void prim_dirOf(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; Path dir = dirOf(state.coerceToPath(*args[0], context)); if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } @@ -580,7 +557,7 @@ static void prim_dirOf(EvalState & state, Value * * args, Value & v) /* Return the contents of a file as a string. */ static void prim_readFile(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; Path path = state.coerceToPath(*args[0], context); if (!context.empty()) throw EvalError(format("string `%1%' cannot refer to other paths") % path); @@ -599,7 +576,7 @@ static void prim_readFile(EvalState & state, Value * * args, Value & v) static void prim_toXML(EvalState & state, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + ContextEntrySet context; printValueAsXML(state, true, false, *args[0], out, context); mkString(v, out.str(), context); } @@ -609,16 +586,15 @@ static void prim_toXML(EvalState & state, Value * * args, Value & v) as an input by derivations. */ static void prim_toFile(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; string name = state.forceStringNoCtx(*args[0]); string contents = state.forceString(*args[1], context); PathSet refs; - foreach (PathSet::iterator, i, context) { - Path path = *i; - if (path.at(0) == '=') path = string(path, 1); - if (isDerivation(path)) + foreach (ContextEntrySet::iterator, i, context) { + Path path = (*i)->path; + if (isDerivation(path) && !(*i)->discardOutputs) throw EvalError(format("in `toFile': the file `%1%' cannot refer to derivation outputs") % name); refs.insert(path); } @@ -631,7 +607,7 @@ static void prim_toFile(EvalState & state, Value * * args, Value & v) result, since `storePath' itself has references to the paths used in args[1]. */ - mkString(v, storePath, singleton(storePath)); + mkString(v, storePath, singleton(state.allocContextEntry(storePath))); } @@ -676,7 +652,7 @@ struct FilterFromExpr : PathFilter static void prim_filterSource(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; Path path = state.coerceToPath(*args[1], context); if (!context.empty()) throw EvalError(format("string `%1%' cannot refer to other paths") % path); @@ -691,7 +667,7 @@ static void prim_filterSource(EvalState & state, Value * * args, Value & v) ? computeStorePathForPath(path, true, htSHA256, filter).first : store->addToStore(path, true, htSHA256, filter); - mkString(v, dstPath, singleton(dstPath)); + mkString(v, dstPath, singleton(state.allocContextEntry(dstPath))); } @@ -1027,7 +1003,7 @@ static void prim_lessThan(EvalState & state, Value * * args, Value & v) `"/nix/store/whatever..."'. */ static void prim_toString(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; string s = state.coerceToString(*args[0], context, true, false); mkString(v, s, context); } @@ -1041,7 +1017,7 @@ static void prim_substring(EvalState & state, Value * * args, Value & v) { int start = state.forceInt(*args[0]); int len = state.forceInt(*args[1]); - PathSet context; + ContextEntrySet context; string s = state.coerceToString(*args[2], context); if (start < 0) throw EvalError("negative start position in `substring'"); @@ -1052,7 +1028,7 @@ static void prim_substring(EvalState & state, Value * * args, Value & v) static void prim_stringLength(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; string s = state.coerceToString(*args[0], context); mkInt(v, s.size()); } @@ -1060,9 +1036,9 @@ static void prim_stringLength(EvalState & state, Value * * args, Value & v) static void prim_unsafeDiscardStringContext(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; string s = state.coerceToString(*args[0], context); - mkString(v, s, PathSet()); + mkString(v, s); } @@ -1074,16 +1050,21 @@ static void prim_unsafeDiscardStringContext(EvalState & state, Value * * args, V drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, Value * * args, Value & v) { - PathSet context; + ContextEntrySet context; string s = state.coerceToString(*args[0], context); - PathSet context2; - foreach (PathSet::iterator, i, context) { - Path p = *i; - if (p.at(0) == '=') p = "~" + string(p, 1); - context2.insert(p); + ContextEntrySet context2; + foreach (ContextEntrySet::iterator, i, context) { + Path path = (*i)->path; + string output = (*i)->output; + if (isDerivation(path) && output.empty()) { + ContextEntry * c = state.allocContextEntry(path,output); + c->discardOutputs = true; + context2.insert(c); + } else + context2.insert(*i); } - + mkString(v, s, context2); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 3934a83eec2..d7f5abd7a62 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -18,7 +18,7 @@ static XMLAttrs singletonAttrs(const string & name, const string & value) static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); + Value & v, XMLWriter & doc, ContextEntrySet & context, PathSet & drvsSeen); static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) @@ -30,7 +30,7 @@ static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Bindings & attrs, XMLWriter & doc, ContextEntrySet & context, PathSet & drvsSeen) { StringSet names; @@ -52,7 +52,7 @@ static void showAttrs(EvalState & state, bool strict, bool location, static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Value & v, XMLWriter & doc, ContextEntrySet & context, PathSet & drvsSeen) { checkInterrupt(); @@ -151,7 +151,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context) + Value & v, std::ostream & out, ContextEntrySet & context) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327edb..1a3c8a79ec9 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -9,6 +9,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context); + Value & v, std::ostream & out, ContextEntrySet & context); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index c9ec236c470..ea806863e0f 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,6 +1,7 @@ #pragma once #include "symbol-table.hh" +#include "context.hh" namespace nix { @@ -61,7 +62,7 @@ struct Value For canonicity, the store paths should be in sorted order. */ struct { const char * s; - const char * * context; // must be in sorted order + const ContextEntry * * context; } string; const char * path; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bd63ce55d09..c3d0f4e9af7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1045,6 +1045,148 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, } +PathSet LocalStore::querySubstitutableFiles(const PathSet & paths) +{ + PathSet res; + foreach (Paths::iterator, i, settings.substituters) { + if (res.size() == paths.size()) break; + RunningSubstituter & run(runningSubstituters[*i]); + startSubstituter(*i, run); + string s = "haveFile "; + foreach (PathSet::const_iterator, j, paths) + if (res.find(*j) == res.end()) { s += *j; s += " "; } + writeLine(run.to, s); + while (true) { + /* FIXME: we only read stderr when an error occurs, so + substituters should only write (short) messages to + stderr when they fail. I.e. they shouldn't write debug + output. */ + try { + Path path = readLine(run.from); + if (path == "") break; + res.insert(path); + } catch (EndOfFile e) { + throw Error(format("substituter `%1%' failed: %2%") % *i % chomp(drainFD(run.error))); + } + } + } + return res; +} + + +void LocalStore::querySubstitutableFileInfos(const Path & substituter, + PathSet & paths, SubstitutableFileInfos & infos) +{ + RunningSubstituter & run(runningSubstituters[substituter]); + startSubstituter(substituter, run); + + string s = "fileInfo "; + foreach (PathSet::const_iterator, i, paths) + if (infos.find(*i) == infos.end()) { s += *i; s += " "; } + writeLine(run.to, s); + + while (true) { + try { + Path path = readLine(run.from); + if (path == "") break; + if (paths.find(path) == paths.end()) + throw Error(format("got unexpected path `%1%' from substituter") % path); + paths.erase(path); + SubstitutableFileInfo & info(infos[path]); + info.recursiveHash = parseHash16or32(htSHA256, readLine(run.from)); + string type = readLine(run.from); + if (type == "regular") { + info.type = tpRegular; + if (readLine(run.from) == "executable") + info.regular.executable = true; + else + info.regular.executable = false; + info.regular.length = getIntLine(run.from); + info.regular.flatHash = parseHash16or32(htSHA256, readLine(run.from)); + } else if (type == "symlink") { + info.type = tpSymlink; + info.target = readLine(run.from); + } else if (type == "directory") { + info.type = tpDirectory; + int nrEntries = getIntLine(run.from); + while (nrEntries--) + info.files.insert(readLine(run.from)); + } else + info.type = tpUnknown; + } catch (EndOfFile e) { + throw Error(format("substituter `%1%' failed: %2%") % substituter % chomp(drainFD(run.error))); + } + } +} + + +void LocalStore::querySubstitutableFileInfos(const PathSet & paths, + SubstitutableFileInfos & infos) +{ + PathSet todo = paths; + foreach (Paths::iterator, i, settings.substituters) { + if (todo.empty()) break; + querySubstitutableFileInfos(*i, todo, infos); + } +} + + +void LocalStore::readSubstitutableFile(const Path & path, FdPair & fds) +{ + bool foundSub; + foreach (Paths::iterator, i, settings.substituters) { + debug(format("starting substituter program `%1%'") % *i); + + Pipe fromPipe, errorPipe; + + fromPipe.create(); + errorPipe.create(); + + int pid = fork(); + + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: /* child */ + try { + /* Pass configuration options (including those overriden + with --option) to the substituter. */ + setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); + + fromPipe.readSide.close(); + errorPipe.readSide.close(); + if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) + throw SysError("dupping stderr"); + closeMostFDs(set()); + execl(i->c_str(), i->c_str(), "--read", path.c_str(), NULL); + throw SysError(format("executing `%1%'") % *i); + } catch (std::exception & e) { + std::cerr << "error: " << e.what() << std::endl; + } + quickExit(1); + } + + /* Parent. */ + + fromPipe.writeSide.close(); + errorPipe.writeSide.close(); + char buf; + if (!read(fromPipe.readSide,&buf,sizeof buf)) + continue; + fds.first = fromPipe.readSide.borrow(); + fds.second = errorPipe.readSide.borrow(); + foundSub = true; + break; + } + if (!foundSub) + throw Error(format("Could not read `%1%' from any substituter") % path); +} + + Hash LocalStore::queryPathHash(const Path & path) { return queryPathInfo(path).hash; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ba058292212..442a71e5955 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -131,6 +131,16 @@ public: void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos); + PathSet querySubstitutableFiles(const PathSet & paths); + + void querySubstitutableFileInfos(const Path & substituter, + PathSet & paths, SubstitutableFileInfos & infos); + + void querySubstitutableFileInfos(const PathSet & paths, + SubstitutableFileInfos & infos); + + void readSubstitutableFile(const Path & path, FdPair & fds); + Path addToStore(const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d3c05f0df4b..790c4c58693 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -14,7 +14,7 @@ #include #include #include - +#include namespace nix { @@ -27,6 +27,39 @@ Path readStorePath(Source & from) } +static int readFD(FdSource & from) +{ + struct msghdr msg = {0}; + struct cmsghdr *cmsg; + struct iovec iov; + int res; + unsigned char control[CMSG_SPACE(sizeof res)]; + + char buf; + iov.iov_base = &buf; + iov.iov_len = sizeof buf; + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = control; + msg.msg_controllen = sizeof control; + + errno = 0; + ssize_t ret = recvmsg(from.fd, &msg, 0); + if (ret <= 0 || + msg.msg_flags & MSG_EOR || + msg.msg_flags & MSG_OOB || + msg.msg_flags & MSG_TRUNC || + msg.msg_flags & MSG_CTRUNC) { + if (errno == EINTR) return readFD(from); + throw SysError("reading fd from socket"); + } + cmsg = CMSG_FIRSTHDR(&msg); + memcpy(&res, CMSG_DATA(cmsg), sizeof res); + return res; +} + + template T readStorePaths(Source & from) { T paths = readStrings(from); @@ -320,6 +353,74 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, } +PathSet RemoteStore::querySubstitutableFiles(const PathSet & paths) +{ + openConnection(); + + if (GET_PROTOCOL_MINOR(daemonVersion) < 13) return PathSet(); + + writeInt(wopQuerySubstitutableFiles, to); + writeStrings(paths, to); + processStderr(); + return readStrings(from); +} + + +void RemoteStore::querySubstitutableFileInfos(const PathSet & paths, + SubstitutableFileInfos & infos) +{ + if (paths.empty()) return; + + openConnection(); + + if (GET_PROTOCOL_MINOR(daemonVersion) < 13) return; + + writeInt(wopQuerySubstitutableFileInfos, to); + writeStrings(paths, to); + processStderr(); + unsigned int count = readInt(from); + for (unsigned int n = 0; n < count; n++) { + Path path = readString(from); + SubstitutableFileInfo & info(infos[path]); + info.recursiveHash = parseHash(htSHA256, readString(from)); + string type = readString(from); + if (type == "regular") { + info.type = tpRegular; + info.regular.executable = readInt(from) != 0; + info.regular.length = readLongLong(from); + info.regular.flatHash = parseHash(htSHA256, readString(from)); + } else if (type == "symlink") { + info.type = tpSymlink; + info.target = readString(from); + } else if (type == "directory") { + info.type = tpDirectory; + int nrEntries = readInt(from); + while (nrEntries--) + info.files.insert(readString(from)); + } else + info.type = tpUnknown; + } +} + + +void RemoteStore::readSubstitutableFile(const Path & path, FdPair & fds) { + openConnection(); + + assert(GET_PROTOCOL_MINOR(daemonVersion) >= 13); + + writeInt(wopReadSubstitutableFile, to); + writeString(path, to); + processStderr(); + /* Needed to ensure delay of writing fd until after stderr is processed + readUnbuffered reads the full bufSize, which could read the dummy byte + that accompanies the written fd, discarding it. */ + writeInt(0,to); + to.flush(); + fds.first = readFD(from); + fds.second = readFD(from); +} + + ValidPathInfo RemoteStore::queryPathInfo(const Path & path) { openConnection(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index ae4c48dad6a..872f97955c3 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -51,6 +51,13 @@ public: void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos); + PathSet querySubstitutableFiles(const PathSet & paths); + + void querySubstitutableFileInfos(const PathSet & paths, + SubstitutableFileInfos & infos); + + void readSubstitutableFile(const Path & path, FdPair & fds); + Path addToStore(const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 32aaca6be0e..3bfd12f7964 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -214,6 +214,38 @@ std::pair computeStorePathForPath(const Path & srcPath, } +std::pair computeStorePathForSubstitutablePath(const Path & srcPath, bool recursive) +{ + SubstitutableFileInfos infos; + store->querySubstitutableFileInfos(singleton(srcPath), infos); + if (!infos.count(srcPath)) + throw Error(format( + "cannot compute store path for `%1%' because its parent store path is invalid and it has no file substitute") + % srcPath); + SubstitutableFileInfo info = infos[srcPath]; + Hash h; + if (recursive) + h = info.recursiveHash; + else { + if (info.type == tpRegular) + h = info.regular.flatHash; + else if (info.type == tpSymlink) { + Path path = absPath(info.target, dirOf(srcPath)); + if (!isInStore(path) || store->isValidPath(toStorePath(path))) + return computeStorePathForPath(path, recursive); + else + return computeStorePathForSubstitutablePath(path, recursive); + } else + throw Error(format( + "cannot compute flat hash of `%1' because it is a directory") + % srcPath); + } + string name = baseNameOf(srcPath); + Path dstPath = makeFixedOutputPath(recursive, htSHA256, h, name); + return std::pair(dstPath, h); +} + + Path computeStorePathForText(const string & name, const string & s, const PathSet & references) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a562360ce34..8117517e790 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,6 +2,7 @@ #include "hash.hh" #include "serialise.hh" +#include "util.hh" #include #include @@ -83,6 +84,36 @@ struct SubstitutablePathInfo typedef std::map SubstitutablePathInfos; +enum FileType { + tpUnknown, + tpRegular, + tpDirectory, + tpSymlink +}; + + +struct SubstitutableFileInfo +{ + Hash recursiveHash; + FileType type; + + // !!! Want a union here, but requires c++11 + struct { + bool executable; + unsigned long long length; + Hash flatHash; + } regular; + + Path target; + + StringSet files; +}; + +typedef std::map SubstitutableFileInfos; + +typedef std::pair FdPair; + + struct ValidPathInfo { Path path; @@ -151,7 +182,20 @@ public: info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) = 0; - + + /* Query which of the given paths have file substitutes. */ + virtual PathSet querySubstitutableFiles(const PathSet & paths) = 0; + + /* Query file substitute info (i.e. file type, length, hash, + etc.) of a set of paths. If a path does not have file substitute + info, it's omitted from the resulting ‘infos’ map. */ + virtual void querySubstitutableFileInfos(const PathSet & paths, + SubstitutableFileInfos & infos) = 0; + + /* Read a file substitute of a path. + Returns the stdout and stderr fds of the substituter in fds */ + virtual void readSubstitutableFile(const Path & path, FdPair & fds) = 0; + /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see @@ -292,6 +336,11 @@ std::pair computeStorePathForPath(const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter); +/* computeStorePathForPath when the path is part of an unrealized store path + but a file substituter may have relevant information about it */ +std::pair computeStorePathForSubstitutablePath(const Path & srcPath, + bool recursive = true); + /* Preparatory part of addTextToStore(). !!! Computation of the path should take the references given to diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 7e4c3ec5fbd..07efc8b05d9 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x10c +#define PROTOCOL_VERSION 0x10d #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -42,6 +42,9 @@ typedef enum { wopQuerySubstitutablePathInfos = 30, wopQueryValidPaths = 31, wopQuerySubstitutablePaths = 32, + wopQuerySubstitutableFiles = 33, + wopQuerySubstitutableFileInfos = 34, + wopReadSubstitutableFile = 35, } WorkerOp; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 5174daf90d8..218a646bee2 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -173,7 +173,7 @@ static void loadDerivations(EvalState & state, Path nixExprPath, system. */ for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { j = i; j++; - if (systemFilter != "*" && i->system != systemFilter) + if (systemFilter != "*" && i->querySystem(state) != systemFilter) elems.erase(i); } } @@ -217,9 +217,13 @@ static bool isPrebuilt(EvalState & state, const DrvInfo & elem) { assert(false); #if 0 - return - store->isValidPath(elem.queryOutPath(state)) || - store->hasSubstitutes(elem.queryOutPath(state)); + try { + return + store->isValidPath(elem.queryOutPath(state)) || + store->hasSubstitutes(elem.queryOutPath(state)); + } catch (ImportReadOnlyError & e) { + return false; + } #endif } @@ -241,7 +245,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, for (DrvInfos::const_iterator j = allElems.begin(); j != allElems.end(); ++j, ++n) { - DrvName drvName(j->name); + DrvName drvName(j->queryName(state)); if (i->matches(drvName)) { i->hits++; matches.push_back(std::pair(*j, n)); @@ -263,35 +267,35 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, StringSet multiple; for (Matches::iterator j = matches.begin(); j != matches.end(); ++j) { - DrvName drvName(j->first.name); + DrvName drvName(j->first.queryName(state)); int d = 1; Newest::iterator k = newest.find(drvName.name); if (k != newest.end()) { - d = j->first.system == k->second.first.system ? 0 : - j->first.system == settings.thisSystem ? 1 : - k->second.first.system == settings.thisSystem ? -1 : 0; + d = j->first.querySystem(state) == k->second.first.querySystem(state) ? 0 : + j->first.querySystem(state) == settings.thisSystem ? 1 : + k->second.first.querySystem(state) == settings.thisSystem ? -1 : 0; if (d == 0) d = comparePriorities(state, j->first, k->second.first); if (d == 0) - d = compareVersions(drvName.version, DrvName(k->second.first.name).version); + d = compareVersions(drvName.version, DrvName(k->second.first.queryName(state)).version); } if (d > 0) { newest[drvName.name] = *j; - multiple.erase(j->first.name); + multiple.erase(j->first.queryName(state)); } else if (d == 0) { - multiple.insert(j->first.name); + multiple.insert(j->first.queryName(state)); } } matches.clear(); for (Newest::iterator j = newest.begin(); j != newest.end(); ++j) { - if (multiple.find(j->second.first.name) != multiple.end()) + if (multiple.find(j->second.first.queryName(state)) != multiple.end()) printMsg(lvlInfo, format("warning: there are multiple derivations named `%1%'; using the first one") - % j->second.first.name); + % j->second.first.queryName(state)); matches.push_back(j->second); } } @@ -395,7 +399,7 @@ static void queryInstSources(EvalState & state, } else elem.setOutPath(path); - elem.name = name; + elem.setName(name); elems.push_back(elem); } @@ -469,8 +473,8 @@ static void installDerivations(Globals & globals, path is not the one we want (e.g., `java-front' versus `java-front-0.9pre15899'). */ if (globals.forceName != "") - i->name = globals.forceName; - newNames.insert(DrvName(i->name).name); + i->setName(globals.forceName); + newNames.insert(DrvName(i->queryName(globals.state)).name); } /* Add in the already installed derivations, unless they have the @@ -483,19 +487,19 @@ static void installDerivations(Globals & globals, DrvInfos allElems(newElems); foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); + DrvName drvName(i->queryName(globals.state)); MetaInfo meta = i->queryMetaInfo(globals.state); if (!globals.preserveInstalled && newNames.find(drvName.name) != newNames.end() && !keep(meta)) printMsg(lvlInfo, - format("replacing old `%1%'") % i->name); + format("replacing old `%1%'") % i->queryName(globals.state)); else allElems.push_back(*i); } foreach (DrvInfos::iterator, i, newElems) - printMsg(lvlInfo, format("installing `%1%'") % i->name); + printMsg(lvlInfo, format("installing `%1%'") % i->queryName(globals.state)); printMissing(globals.state, newElems); @@ -547,7 +551,7 @@ static void upgradeDerivations(Globals & globals, /* Go through all installed derivations. */ DrvInfos newElems; foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); + DrvName drvName(i->queryName(globals.state)); try { @@ -566,7 +570,7 @@ static void upgradeDerivations(Globals & globals, DrvInfos::iterator bestElem = availElems.end(); DrvName bestName; foreach (DrvInfos::iterator, j, availElems) { - DrvName newName(j->name); + DrvName newName(j->queryName(globals.state)); if (newName.name == drvName.name) { int d = comparePriorities(globals.state, *i, *j); if (d == 0) d = compareVersions(drvName.version, newName.version); @@ -594,12 +598,12 @@ static void upgradeDerivations(Globals & globals, { printMsg(lvlInfo, format("upgrading `%1%' to `%2%'") - % i->name % bestElem->name); + % i->queryName(globals.state) % bestElem->queryName(globals.state)); newElems.push_back(*bestElem); } else newElems.push_back(*i); } catch (Error & e) { - e.addPrefix(format("while trying to find an upgrade for `%1%':\n") % i->name); + e.addPrefix(format("while trying to find an upgrade for `%1%':\n") % i->queryName(globals.state)); throw; } } @@ -664,11 +668,11 @@ static void opSetFlag(Globals & globals, /* Update all matching derivations. */ foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); + DrvName drvName(i->queryName(globals.state)); foreach (DrvNames::iterator, j, selectors) if (j->matches(drvName)) { printMsg(lvlInfo, - format("setting flag on `%1%'") % i->name); + format("setting flag on `%1%'") % i->queryName(globals.state)); setMetaFlag(globals.state, *i, flagName, flagValue); break; } @@ -726,7 +730,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, DrvInfos newElems; foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); + DrvName drvName(i->queryName(globals.state)); bool found = false; foreach (Strings::iterator, j, selectors) /* !!! the repeated calls to followLinksToStorePath() @@ -734,7 +738,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, if ((isPath(*j) && i->queryOutPath(globals.state) == followLinksToStorePath(*j)) || DrvName(*j).matches(drvName)) { - printMsg(lvlInfo, format("uninstalling `%1%'") % i->name); + printMsg(lvlInfo, format("uninstalling `%1%'") % i->queryName(globals.state)); found = true; break; } @@ -764,12 +768,19 @@ static bool cmpChars(char a, char b) } -static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b) +class cmpElemByName { - return lexicographical_compare( - a.name.begin(), a.name.end(), - b.name.begin(), b.name.end(), cmpChars); -} + public: + cmpElemByName(EvalState & state) : _state(state) { } + bool operator()(const DrvInfo & a, const DrvInfo & b) + { + return lexicographical_compare( + a.queryName(_state).begin(), a.queryName(_state).end(), + b.queryName(_state).begin(), b.queryName(_state).end(), cmpChars); + } + private: + EvalState & _state; +}; typedef list Table; @@ -814,16 +825,16 @@ void printTable(Table & table) typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff; -static VersionDiff compareVersionAgainstSet( +static VersionDiff compareVersionAgainstSet(EvalState & state, const DrvInfo & elem, const DrvInfos & elems, string & version) { - DrvName name(elem.name); + DrvName name(elem.queryName(state)); VersionDiff diff = cvUnavail; version = "?"; for (DrvInfos::const_iterator i = elems.begin(); i != elems.end(); ++i) { - DrvName name2(i->name); + DrvName name2(i->queryName(state)); if (name.name == name2.name) { int d = compareVersions(name.version, name2.version); if (d < 0) { @@ -921,9 +932,10 @@ static void opQuery(Globals & globals, /* Sort them by name. */ /* !!! */ vector elems2; + cmpElemByName nameComparer(globals.state); for (DrvInfos::iterator i = elems.begin(); i != elems.end(); ++i) elems2.push_back(*i); - sort(elems2.begin(), elems2.end(), cmpElemByName); + sort(elems2.begin(), elems2.end(), nameComparer); /* We only need to know the installed paths when we are querying @@ -941,13 +953,18 @@ static void opQuery(Globals & globals, PathSet validPaths, substitutablePaths; if (printStatus) { PathSet paths; - foreach (vector::iterator, i, elems2) + foreach (vector::iterator, i, elems2) { + string name; try { + name = i->queryName(globals.state); paths.insert(i->queryOutPath(globals.state)); } catch (AssertionError & e) { - printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name); + printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % name); i->setFailed(); + } catch (ImportReadOnlyError & e) { + printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an ImportReadOnly failure") % name); } + } validPaths = store->queryValidPaths(paths); substitutablePaths = store->querySubstitutablePaths(paths); } @@ -974,19 +991,23 @@ static void opQuery(Globals & globals, XMLAttrs attrs; if (printStatus) { - Path outPath = i->queryOutPath(globals.state); - bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); - bool isInstalled = installed.find(outPath) != installed.end(); - bool isValid = validPaths.find(outPath) != validPaths.end(); - if (xmlOutput) { - attrs["installed"] = isInstalled ? "1" : "0"; - attrs["valid"] = isValid ? "1" : "0"; - attrs["substitutable"] = hasSubs ? "1" : "0"; - } else - columns.push_back( - (string) (isInstalled ? "I" : "-") - + (isValid ? "P" : "-") - + (hasSubs ? "S" : "-")); + try { + Path outPath = i->queryOutPath(globals.state); + bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); + bool isInstalled = installed.find(outPath) != installed.end(); + bool isValid = validPaths.find(outPath) != validPaths.end(); + if (xmlOutput) { + attrs["installed"] = isInstalled ? "1" : "0"; + attrs["valid"] = isValid ? "1" : "0"; + attrs["substitutable"] = hasSubs ? "1" : "0"; + } else + columns.push_back( + (string) (isInstalled ? "I" : "-") + + (isValid ? "P" : "-") + + (hasSubs ? "S" : "-")); + } catch (ImportReadOnlyError & e) { + if (!xmlOutput) columns.push_back("?"); + } } if (xmlOutput) @@ -995,9 +1016,9 @@ static void opQuery(Globals & globals, columns.push_back(i->attrPath); if (xmlOutput) - attrs["name"] = i->name; + attrs["name"] = i->queryName(globals.state); else if (printName) - columns.push_back(i->name); + columns.push_back(i->queryName(globals.state)); if (compareVersions) { /* Compare this element against the versions of the @@ -1005,7 +1026,7 @@ static void opQuery(Globals & globals, elements, or the set of installed elements. !!! This is O(N * M), should be O(N * lg M). */ string version; - VersionDiff diff = compareVersionAgainstSet(*i, otherElems, version); + VersionDiff diff = compareVersionAgainstSet(globals.state, *i, otherElems, version); char ch; switch (diff) { @@ -1029,62 +1050,75 @@ static void opQuery(Globals & globals, } if (xmlOutput) { - if (i->system != "") attrs["system"] = i->system; + if (i->querySystem(globals.state) != "") attrs["system"] = i->querySystem(globals.state); } else if (printSystem) - columns.push_back(i->system); + columns.push_back(i->querySystem(globals.state)); if (printDrvPath) { - string drvPath = i->queryDrvPath(globals.state); - if (xmlOutput) { - if (drvPath != "") attrs["drvPath"] = drvPath; - } else - columns.push_back(drvPath == "" ? "-" : drvPath); + try { + string drvPath = i->queryDrvPath(globals.state); + if (xmlOutput) { + if (drvPath != "") attrs["drvPath"] = drvPath; + } else + columns.push_back(drvPath == "" ? "-" : drvPath); + } catch (ImportReadOnlyError & e) { + if (!xmlOutput) columns.push_back("?"); + } } if (printOutPath) { - string outPath = i->queryOutPath(globals.state); - if (xmlOutput) { - if (outPath != "") attrs["outPath"] = outPath; - } else - columns.push_back(outPath); + try { + string outPath = i->queryOutPath(globals.state); + if (xmlOutput) { + if (outPath != "") attrs["outPath"] = outPath; + } else + columns.push_back(outPath); + } catch (ImportReadOnlyError & e) { + if (!xmlOutput) columns.push_back("?"); + } } if (printDescription) { - MetaInfo meta = i->queryMetaInfo(globals.state); - MetaValue value = meta["description"]; - string descr = value.type == MetaValue::tpString ? value.stringValue : ""; - if (xmlOutput) { - if (descr != "") attrs["description"] = descr; - } else - columns.push_back(descr); + try { + MetaValue value = i->queryMetaInfo(globals.state, "description"); + string descr = value.type == MetaValue::tpString ? value.stringValue : ""; + if (xmlOutput) { + if (descr != "") attrs["description"] = descr; + } else + columns.push_back(descr); + } catch (ImportReadOnlyError & e) { + if (!xmlOutput) columns.push_back("?"); + } } if (xmlOutput) if (printMeta) { XMLOpenElement item(xml, "item", attrs); - MetaInfo meta = i->queryMetaInfo(globals.state); - for (MetaInfo::iterator j = meta.begin(); j != meta.end(); ++j) { - XMLAttrs attrs2; - attrs2["name"] = j->first; - if (j->second.type == MetaValue::tpString) { - attrs2["type"] = "string"; - attrs2["value"] = j->second.stringValue; - xml.writeEmptyElement("meta", attrs2); - } else if (j->second.type == MetaValue::tpInt) { - attrs2["type"] = "int"; - attrs2["value"] = (format("%1%") % j->second.intValue).str(); - xml.writeEmptyElement("meta", attrs2); - } else if (j->second.type == MetaValue::tpStrings) { - attrs2["type"] = "strings"; - XMLOpenElement m(xml, "meta", attrs2); - foreach (Strings::iterator, k, j->second.stringValues) { - XMLAttrs attrs3; - attrs3["value"] = *k; - xml.writeEmptyElement("string", attrs3); - } + try { + MetaInfo meta = i->queryMetaInfo(globals.state); + for (MetaInfo::iterator j = meta.begin(); j != meta.end(); ++j) { + XMLAttrs attrs2; + attrs2["name"] = j->first; + if (j->second.type == MetaValue::tpString) { + attrs2["type"] = "string"; + attrs2["value"] = j->second.stringValue; + xml.writeEmptyElement("meta", attrs2); + } else if (j->second.type == MetaValue::tpInt) { + attrs2["type"] = "int"; + attrs2["value"] = (format("%1%") % j->second.intValue).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (j->second.type == MetaValue::tpStrings) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + foreach (Strings::iterator, k, j->second.stringValues) { + XMLAttrs attrs3; + attrs3["value"] = *k; + xml.writeEmptyElement("string", attrs3); + } + } } - } + } catch (ImportReadOnlyError & e) { } } else xml.writeEmptyElement("item", attrs); @@ -1094,7 +1128,7 @@ static void opQuery(Globals & globals, cout.flush(); } catch (AssertionError & e) { - printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name); + printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->queryName(globals.state)); } } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 510c5ca3817..e69ec28e0f5 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -11,7 +11,7 @@ namespace nix { -static void readLegacyManifest(const Path & path, DrvInfos & elems); +static void readLegacyManifest(EvalState & state, const Path & path, DrvInfos & elems); DrvInfos queryInstalled(EvalState & state, const Path & userEnv) @@ -27,7 +27,7 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv) Bindings bindings; getDerivations(state, v, "", bindings, elems); } else if (pathExists(oldManifestFile)) - readLegacyManifest(oldManifestFile, elems); + readLegacyManifest(state, oldManifestFile, elems); return elems; } @@ -63,8 +63,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, state.mkAttrs(v, 8); mkString(*state.allocAttr(v, state.sType), "derivation"); - mkString(*state.allocAttr(v, state.sName), i->name); - mkString(*state.allocAttr(v, state.sSystem), i->system); + mkString(*state.allocAttr(v, state.sName), i->queryName(state)); + mkString(*state.allocAttr(v, state.sSystem), i->querySystem(state)); mkString(*state.allocAttr(v, state.sOutPath), i->queryOutPath(state)); if (drvPath != "") mkString(*state.allocAttr(v, state.sDrvPath), i->queryDrvPath(state)); @@ -119,7 +119,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, Value args, topLevel; state.mkAttrs(args, 3); mkString(*state.allocAttr(args, state.symbols.create("manifest")), - manifestFile, singleton(manifestFile)); + manifestFile, singleton(state.allocContextEntry(manifestFile))); args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest)); args.attrs->sort(); mkApp(topLevel, envBuilder, args); @@ -214,7 +214,7 @@ static MetaInfo parseMeta(std::istream & str) } -static void readLegacyManifest(const Path & path, DrvInfos & elems) +static void readLegacyManifest(EvalState & state, const Path & path, DrvInfos & elems) { string manifest = readFile(path); std::istringstream str(manifest); @@ -234,10 +234,10 @@ static void readLegacyManifest(const Path & path, DrvInfos & elems) if (name == "meta") elem.setMetaInfo(parseMeta(str)); else { string value = parseStr(str); - if (name == "name") elem.name = value; + if (name == "name") elem.setName(value); else if (name == "outPath") elem.setOutPath(value); else if (name == "drvPath") elem.setDrvPath(value); - else if (name == "system") elem.system = value; + else if (name == "system") elem.setSystem(value); } expect(str, ",NoPos)"); @@ -245,7 +245,7 @@ static void readLegacyManifest(const Path & path, DrvInfos & elems) expect(str, ")"); - if (elem.name != "") { + if (elem.queryName(state) != "") { elem.attrPath = int2String(n++); elems.push_back(elem); } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 6ddf9f069ef..bdc89547006 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -47,7 +47,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, findAlongAttrPath(state, *i, autoArgs, e, v); state.forceValue(v); - PathSet context; + ContextEntrySet context; if (evalOnly) if (xmlOutput) printValueAsXML(state, strict, location, v, std::cout, context); diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index dadde9cc518..955316204c4 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -274,8 +274,43 @@ struct SavingSourceAdapter : Source }; +static void writeFD(const AutoCloseFD & fd, FdSink & to) +{ + struct msghdr msg = {0}; + struct cmsghdr * cmsg; + struct iovec iov; + int realfd = fd; + unsigned char control[CMSG_SPACE(sizeof realfd)]; + + char buf = '!'; + iov.iov_base = &buf; + iov.iov_len = sizeof buf; + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = control; + msg.msg_controllen = sizeof control; + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_len = CMSG_LEN(sizeof realfd); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + memcpy(CMSG_DATA(cmsg), &realfd, sizeof realfd); + + to.flush(); + ssize_t ret = sendmsg(to.fd, &msg, 0); + if (ret == -1) { + if (errno == EINTR) { + writeFD(fd,to); + return; + } + throw SysError("writing fd to socket"); + } +} + + static void performOp(unsigned int clientVersion, - Source & from, Sink & to, unsigned int op) + FdSource & from, FdSink & to, unsigned int op) { switch (op) { @@ -632,6 +667,64 @@ static void performOp(unsigned int clientVersion, break; } + case wopQuerySubstitutableFiles: { + PathSet paths = readStrings(from); + startWork(); + PathSet res = store->querySubstitutableFiles(paths); + stopWork(); + writeStrings(res, to); + break; + } + + case wopQuerySubstitutableFileInfos: { + PathSet paths = readStrings(from); + startWork(); + SubstitutableFileInfos infos; + store->querySubstitutableFileInfos(paths, infos); + stopWork(); + writeInt(infos.size(), to); + foreach (SubstitutableFileInfos::iterator, i, infos) { + writeString(i->first, to); + writeString(printHash(i->second.recursiveHash), to); + switch (i->second.type) + { + case tpRegular: + writeString("regular", to); + writeInt(i->second.regular.executable, to); + writeLongLong(i->second.regular.length, to); + writeString(printHash(i->second.regular.flatHash), to); + break; + case tpSymlink: + writeString("symlink", to); + writeString(i->second.target, to); + break; + case tpDirectory: + writeString("directory", to); + writeStrings(i->second.files, to); + break; + case tpUnknown: + writeString("unknown", to); + break; + } + } + break; + } + + case wopReadSubstitutableFile: { + Path path = readString(from); + FdPair fds; + startWork(); + store->readSubstitutableFile(path, fds); + stopWork(); + to.flush(); + /* See comment in readSubstitutableFile in remote-store.cc */ + if (readInt(from)) + throw Error("FD-ready int not written!"); + writeFD(fds.first.borrow(),to); + writeFD(fds.second.borrow(),to); + break; + } + default: throw Error(format("invalid operation %1%") % op); } diff --git a/tests/Makefile.am b/tests/Makefile.am index 641e29d7e7a..74c04a5e7ba 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -10,13 +10,13 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ remote-store.sh export.sh export-graph.sh negative-caching.sh \ binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ - binary-cache.sh + binary-cache.sh file-substitutes.sh XFAIL_TESTS = include ../substitute.mk -$(TESTS): common.sh config.nix +$(TESTS): common.sh config.nix file-substituter.sh EXTRA_DIST = $(TESTS) \ config.nix.in \ @@ -40,6 +40,8 @@ EXTRA_DIST = $(TESTS) \ multiple-outputs.nix \ import-derivation.nix \ fetchurl.nix \ + file-substitutes.nix \ + file-substituter.sh.in \ $(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) $(wildcard lang/*.flags) $(wildcard lang/dir*/*.nix) \ common.sh.in diff --git a/tests/file-substituter.sh.in b/tests/file-substituter.sh.in new file mode 100755 index 00000000000..5d007579354 --- /dev/null +++ b/tests/file-substituter.sh.in @@ -0,0 +1,129 @@ +#! /bin/sh -e +echo substituter args: $* >&2 + +nixfile="@abs_top_srcdir@/tests/file-substitutes.nix" +barOutPath=`nix-instantiate --eval-only -A bar.outPath $nixfile | tr -d \"` +barDrvPath=`nix-instantiate --eval-only -A bar.drvPath $nixfile | tr -d \"` +bazOutPath=`nix-instantiate --eval-only -A bar.baz.outPath $nixfile | tr -d \"` +bazDrvPath=`nix-instantiate --eval-only -A bar.baz.drvPath $nixfile | tr -d \"` +substitutedExprPath=`nix-instantiate --eval-only -A substitutedExpr $nixfile | tr -d \"` +substitutedExprString=`echo -e \`nix-instantiate --eval-only -A substitutedExprString $nixfile | tr -d \"\`` +filePath=`nix-instantiate --eval-only -A file $nixfile | tr -d \"` +fileString=`nix-instantiate --eval-only -A fileString $nixfile | tr -d \"` +aDotNixPath=`nix-instantiate --eval-only -A aDotNix $nixfile | tr -d \"` +aDotNixString=`nix-instantiate --eval-only -A aDotNixString $nixfile | tr -d \"` + +if test $1 = "--query"; then + while read cmd args; do + echo "CMD = $cmd, ARGS = $args" >&2 + if test "$cmd" = "have"; then + for path in $args; do + if test "$path" = "$barOutPath" || test "$path" = "$bazOutPath"; then + echo $path + fi + done + echo + elif test "$cmd" = "info"; then + for path in $args; do + if test "$path" = "$barOutPath"; then + echo "$path" + echo "$barDrvPath" + echo 1 + echo "$bazOutPath" + echo $((1 * 1024 * 1024)) + echo $((2 * 1024 * 1024)) + elif test "$path" = "$bazOutPath"; then + echo "$path" + echo "$bazDrvPath" + echo 0 + echo $((1 * 1024 * 1024)) # download size + echo $((2 * 1024 * 1024)) # nar size + fi + done + echo + elif test "$cmd" = "haveFile"; then + for path in $args; do + if test "$path" = "$barOutPath/default.nix"; then + echo "$path" + elif test "$path" = "$bazOutPath/file"; then + echo "$path" + elif test "$path" = "$bazOutPath/a.nix"; then + echo "$path" + fi + done + echo + elif test "$cmd" = "fileInfo"; then + for path in $args; do + if test "$path" = "$barOutPath/default.nix"; then + echo "$path" + echo 1234567890123456789012345678901234567890123456789012345678901230 # File hash, recursive + echo "regular" # File type + echo "" # Not executable ("executable" otherwise) + echo `expr length "$substitutedExprString"` # File length + echo 0123456789012345678901234567890123456789012345678901234567890123 # File hash, flat + elif test "$path" = "$barOutPath"; then + echo "$path" + echo 2345678901234567890123456789012345678901234567890123456789012301 # File hash, recursive + echo "directory" # File type + echo "3" # File count + echo "file" # Entry + echo "a.nix" # Entry + echo "default.nix" # Entry + elif test "$path" = "$bazOutPath"; then + echo "$path" + echo 3456789012345678901234567890123456789012345678901234567890123012 # File hash, recursive + echo "directory" + echo "2" + echo "file" + echo "a.nix" + elif test "$path" = "$barOutPath/file"; then + echo "$path" + echo 4567890123456789012345678901234567890123456789012345678901230123 # File hash, recursive + echo "symlink" # File type + echo "$bazOutPath/file" # Target + elif test "$path" = "$barOutPath/a.nix"; then + echo "$path" + echo 9012345678901234567890123456789012345678901234567890123012345678 # File hash, recursive + echo "symlink" # File type + echo "$bazOutPath/a.nix" # Target + elif test "$path" = "$bazOutPath/file"; then + echo "$path" + echo 5678901234567890123456789012345678901234567890123456789012301234 # File hash, recursive + echo "regular" # File type + echo "" # Not executable ("executable" otherwise) + echo `expr length "$fileString"` + echo 6789012345678901234567890123456789012345678901234567890123012345 # File hash, flat + elif test "$path" = "$bazOutPath/a.nix"; then + echo "$path" + echo 7890123456789012345678901234567890123456789012345678901230123456 # File hash, recursive + echo "regular" + echo "" + echo `expr length "$aDotNixString"` + echo 8901234567890123456789012345678901234567890123456789012301234567 # File hash, flat + fi + done + echo + else + echo "bad command $cmd" + exit 1 + fi + done +elif test $1 = "--read"; then + if test "$2" = "$barOutPath/default.nix"; then + echo # Need to echo a char to indicate we have the file + echo "$substitutedExprString" + elif test "$2" = "$bazOutPath/file"; then + echo + echo "$fileString" + elif test "$2" = "$bazOutPath/a.nix"; then + echo + echo "$aDotNixString" + else + exit 1 + fi +elif test $1 = "--substitute"; then + exit 1 +else + echo "unknown substituter operation" + exit 1 +fi diff --git a/tests/file-substitutes.nix b/tests/file-substitutes.nix new file mode 100644 index 00000000000..67e7d99abb0 --- /dev/null +++ b/tests/file-substitutes.nix @@ -0,0 +1,60 @@ +with import ./config.nix; + +let + failBuilder = builtins.toFile "builder.sh" '' + exit 1 + ''; + + bar = mkDerivation { + name = "bar"; + builder = failBuilder; + baz = mkDerivation { + name = "baz"; + builder = failBuilder; + }; + }; + + bazOutPath = builtins.unsafeDiscardStringContext bar.baz.outPath; + + substitutedExprString = '' + let + failBuilder = builtins.toFile '''builder.sh''' ''' + exit 1 + '''; + in + + { + evalFile1 = import ./a.nix; + evalFile2 = import ${bazOutPath}/a.nix; + + coerceToString1 = (derivation { + name = '''coerceToString'''; + builder = '''''${failBuilder}'''; + system = '''nix'''; + file = ./file; + }).drvPath; + coerceToString2 = (derivation { + name = '''coerceToString'''; + builder = '''''${failBuilder}'''; + system = '''nix'''; + file = ${bazOutPath}/file; + }).drvPath; + } + ''; + + fileString = "A file"; + + aDotNixString = "true"; +in + +{ + substitutedExpr = builtins.toFile "default.nix" substitutedExprString; + + file = builtins.toFile "file" fileString; + + aDotNix = builtins.toFile "a.nix" aDotNixString; + + imported = import bar.outPath; + + inherit bar substitutedExprString fileString aDotNixString; +} diff --git a/tests/file-substitutes.sh b/tests/file-substitutes.sh new file mode 100644 index 00000000000..e066e05681c --- /dev/null +++ b/tests/file-substitutes.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +export NIX_SUBSTITUTERS=$(pwd)/file-substituter.sh + +startDaemon + +evalFile1=`nix-instantiate --eval-only -A imported.evalFile1 $(pwd)/file-substitutes.nix` +evalFile2=`nix-instantiate --eval-only -A imported.evalFile2 $(pwd)/file-substitutes.nix` +coerceToString1=`nix-instantiate --eval-only -A imported.coerceToString1 $(pwd)/file-substitutes.nix` +coerceToString2=`nix-instantiate --eval-only -A imported.coerceToString2 $(pwd)/file-substitutes.nix` + +test "$evalFile1" = "$evalFile2" +test "$evalFile1" = "true"