Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace some bool recursive with a new FileIngestionMethod enum #3455

Merged
merged 11 commits into from
May 28, 2020
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions perl/lib/Nix/Store.xs
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
SV * addToStore(char * srcPath, int recursive, char * algo)
PPCODE:
try {
auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
Expand All @@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
PPCODE:
try {
Hash h(hash, parseHashType(algo));
auto path = store()->makeFixedOutputPath(recursive, h, name);
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name);
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
Expand Down
2 changes: 1 addition & 1 deletion src/libexpr/eval.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1669,7 +1669,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
else {
auto p = settings.readOnlyMode
? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
dstPath = store->printStorePath(p);
srcToStore.insert_or_assign(path, std::move(p));
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
Expand Down
25 changes: 14 additions & 11 deletions src/libexpr/primops.cc
Original file line number Diff line number Diff line change
Expand Up @@ -562,7 +562,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *

std::optional<std::string> outputHash;
std::string outputHashAlgo;
bool outputHashRecursive = false;
auto ingestionMethod = FileIngestionMethod::Flat;

StringSet outputs;
outputs.insert("out");
Expand All @@ -573,8 +573,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
vomit("processing attribute '%1%'", key);

auto handleHashMode = [&](const std::string & s) {
if (s == "recursive") outputHashRecursive = true;
else if (s == "flat") outputHashRecursive = false;
if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName);
};

Expand Down Expand Up @@ -721,11 +721,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
Hash h(*outputHash, ht);

auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
(outputHashRecursive ? "r:" : "") + printHashType(h.type),
h.to_string(Base16, false)));
drv.outputs.insert_or_assign("out", DerivationOutput {
std::move(outPath),
(ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "")
+ printHashType(h.type),
h.to_string(Base16, false),
});
}

else {
Expand Down Expand Up @@ -1039,7 +1042,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu


static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
Value * filterFun, FileIngestionMethod recursive, const Hash & expectedHash, Value & v)
{
const auto path = evalSettings.pureEval && expectedHash ?
path_ :
Expand Down Expand Up @@ -1096,7 +1099,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
if (args[0]->type != tLambda)
throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);

addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v);
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
}

static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
Expand All @@ -1105,7 +1108,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
Path path;
string name;
Value * filterFun = nullptr;
auto recursive = true;
auto recursive = FileIngestionMethod::Recursive;
Hash expectedHash;

for (auto & attr : *args[0]->attrs) {
Expand All @@ -1121,7 +1124,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
state.forceValue(*attr.value, pos);
filterFun = attr.value;
} else if (n == "recursive")
recursive = state.forceBool(*attr.value, *attr.pos);
recursive = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) };
else if (n == "sha256")
expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
else
Expand Down
4 changes: 2 additions & 2 deletions src/libfetchers/git.cc
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ struct GitInput : Input
return files.count(file);
};

auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);

auto tree = Tree {
.actualPath = store->printStorePath(storePath),
Expand Down Expand Up @@ -347,7 +347,7 @@ struct GitInput : Input
unpackTarfile(*source, tmpDir);
}

auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter);
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);

auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));

Expand Down
2 changes: 1 addition & 1 deletion src/libfetchers/mercurial.cc
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ struct MercurialInput : Input
return files.count(file);
};

auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);

return {Tree {
.actualPath = store->printStorePath(storePath),
Expand Down
6 changes: 3 additions & 3 deletions src/libfetchers/tarball.cc
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,10 @@ DownloadFileResult downloadFile(
StringSink sink;
dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data);
ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(false, hash);
info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
storePath = std::move(info.path);
}
Expand Down Expand Up @@ -141,7 +141,7 @@ Tree downloadTarball(
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
lastModified = lstat(topDir).st_mtime;
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
}

Attrs infoAttrs({
Expand Down
2 changes: 1 addition & 1 deletion src/libfetchers/tree-info.cc
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ namespace nix::fetchers {
StorePath TreeInfo::computeStorePath(Store & store) const
{
assert(narHash);
return store.makeFixedOutputPath(true, narHash, "source");
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
}

}
4 changes: 2 additions & 2 deletions src/libstore/binary-cache-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
}

StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
// FIXME: some cut&paste from LocalStore::addToStore().

Expand All @@ -336,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
small files. */
StringSink sink;
Hash h;
if (recursive) {
if (recursive == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s);
} else {
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/binary-cache-store.hh
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public:
std::shared_ptr<FSAccessor> accessor) override;

StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
FileIngestionMethod recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;

StorePath addTextToStore(const string & name, const string & s,
Expand Down
16 changes: 10 additions & 6 deletions src/libstore/build.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2734,7 +2734,7 @@ struct RestrictedStore : public LocalFSStore
{ throw Error("queryPathFromHashPart"); }

StorePath addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
{ throw Error("addToStore"); }

Expand All @@ -2747,7 +2747,7 @@ struct RestrictedStore : public LocalFSStore
}

StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
{
auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair);
goal.addDependency(path);
Expand Down Expand Up @@ -3692,10 +3692,10 @@ void DerivationGoal::registerOutputs()

if (fixedOutput) {

bool recursive; Hash h;
FileIngestionMethod recursive; Hash h;
i.second.parseHashInfo(recursive, h);

if (!recursive) {
if (recursive == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
Expand All @@ -3705,7 +3705,9 @@ void DerivationGoal::registerOutputs()

/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
Hash h2 = recursive == FileIngestionMethod::Recursive
? hashPath(h.type, actualPath).first
: hashFile(h.type, actualPath);

auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name());

Expand Down Expand Up @@ -3957,7 +3959,9 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)

auto spec = parseReferenceSpecifiers(worker.store, *drv, *value);

auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references);
auto used = recursive
? cloneStorePathSet(getClosure(info.path).first)
: cloneStorePathSet(info.references);

if (recursive && checks.ignoreSelfRefs)
used.erase(info.path);
Expand Down
24 changes: 16 additions & 8 deletions src/libstore/daemon.cc
Original file line number Diff line number Diff line change
Expand Up @@ -355,20 +355,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}

case wopAddToStore: {
bool fixed, recursive;
std::string s, baseName;
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
/* Compatibility hack. */
if (!fixed) {
s = "sha256";
recursive = true;
FileIngestionMethod method;
{
bool fixed, recursive;
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
method = FileIngestionMethod { recursive };
/* Compatibility hack. */
if (!fixed) {
s = "sha256";
method = FileIngestionMethod::Recursive;
}
}
HashType hashAlgo = parseHashType(s);

TeeSource savedNAR(from);
RetrieveRegularNARSink savedRegular;

if (recursive) {
if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to
a string so that we can pass it to
addToStoreFromDump(). */
Expand All @@ -380,7 +384,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected");

auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
auto path = store->addToStoreFromDump(
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
baseName,
method,
hashAlgo);
logger->stopWork();

to << store->printStorePath(path);
Expand Down
6 changes: 3 additions & 3 deletions src/libstore/derivations.cc
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@
namespace nix {


void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
{
recursive = false;
recursive = FileIngestionMethod::Flat;
string algo = hashAlgo;

if (string(algo, 0, 2) == "r:") {
recursive = true;
recursive = FileIngestionMethod::Recursive;
algo = string(algo, 2);
}

Expand Down
2 changes: 1 addition & 1 deletion src/libstore/derivations.hh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ struct DerivationOutput
, hashAlgo(std::move(hashAlgo))
, hash(std::move(hash))
{ }
void parseHashInfo(bool & recursive, Hash & hash) const;
void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
};

typedef std::map<string, DerivationOutput> DerivationOutputs;
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/legacy-ssh-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ struct LegacySSHStore : public Store
{ unsupported("queryPathFromHashPart"); }

StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override
{ unsupported("addToStore"); }

Expand Down
12 changes: 6 additions & 6 deletions src/libstore/local-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -557,7 +557,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));

bool recursive; Hash h;
FileIngestionMethod recursive; Hash h;
out->second.parseHashInfo(recursive, h);

check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
Expand Down Expand Up @@ -1043,7 +1043,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,


StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
bool recursive, HashType hashAlgo, RepairFlag repair)
FileIngestionMethod recursive, HashType hashAlgo, RepairFlag repair)
{
Hash h = hashString(hashAlgo, dump);

Expand All @@ -1067,7 +1067,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam

autoGC();

if (recursive) {
if (recursive == FileIngestionMethod::Recursive) {
StringSource source(dump);
restorePath(realPath, source);
} else
Expand All @@ -1080,7 +1080,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
above (if called with recursive == true and hashAlgo ==
sha256); otherwise, compute it here. */
HashResult hash;
if (recursive) {
if (recursive == FileIngestionMethod::Recursive) {
hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
hash.second = dump.size();
} else
Expand All @@ -1103,15 +1103,15 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam


StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
Path srcPath(absPath(_srcPath));

/* Read the whole path into memory. This is not a very scalable
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
if (recursive)
if (recursive == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink, filter);
else
sink.s = make_ref<std::string>(readFile(srcPath));
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/local-store.hh
Original file line number Diff line number Diff line change
Expand Up @@ -149,15 +149,15 @@ public:
std::shared_ptr<FSAccessor> accessor) override;

StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;

/* Like addToStore(), but the contents of the path are contained
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;

StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
Expand Down
Loading