From 5de77536ef3c6c2e5fbda6089249495bbf12efaa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2024 19:58:54 +0200 Subject: [PATCH 1/3] Basic store path provenance tracking Nix historically has been bad at being able to answer the question "where did this store path come from", i.e. to provide traceability from a store path back to the Nix expression from which is was built. Nix tracks the "deriver" of a store path (the .drv file that built it) but that's pretty useless in practice, since it doesn't link back to the Nix expressions. So this PR adds a "provenance" field (a JSON object) to the ValidPaths table and to .narinfo files that describes where the store path came from and how it can be reproduced. There are currently 3 types of provenance: * "copied": Records that the store path was copied or substituted from another store (typically a binary cache). Its "from" field is the URL of the origin store. Its "provenance" field propagates the provenance of the store path on the origin store. * "derivation": Records that the store path is the output of a .drv file. This is equivalent for the "deriver" field, but it has a nested "provenance" field that records how the .drv file was created. * "flake": Records that the store path was created during the evaluation of a flake output. Example: $ nix path-info --json /nix/store/xcqzb13bd60zmfw6wv0z4242b9mfw042-patchelf-0.18.0 { "/nix/store/xcqzb13bd60zmfw6wv0z4242b9mfw042-patchelf-0.18.0": { "provenance": { "from": "https://cache.example.org", "provenance": { "drv": "rlabxgjx88bavjkc694v1bqbwslwivxs-patchelf-0.18.0.drv", "output": "out", "provenance": { "flake": { "lastModified": 1729856604, "narHash": "sha256-obmE2ZI9sTPXczzGMerwQX4SALF+ABL9J0oB371yvZE=", "owner": "NixOS", "repo": "patchelf", "rev": "689f19e499caee8e5c3d387008bbd4ed7f8dc3a9", "type": "github", }, "output": "packages.x86_64-linux.default", "type": "flake" }, "type": "derivation" }, "type": "copied" }, ... } } This specifies that the store path was copied from the binary cache https://cache.example.org and it's the "out" output of a store derivation that was produced by evaluating the flake ouput `packages.x86_64-linux.default` of some revision of the patchelf GitHub repository. --- src/libcmd/installable-flake.cc | 11 ++- src/libexpr/eval.cc | 15 ++- src/libexpr/eval.hh | 10 ++ src/libexpr/primops.cc | 15 ++- src/libfetchers/fetch-to-store.cc | 5 +- src/libfetchers/fetch-to-store.hh | 3 +- src/libstore-test-support/tests/protocol.hh | 4 + src/libstore-tests/data/nar-info/impure.json | 1 + .../data/path-info/empty_impure.json | 1 + src/libstore-tests/data/path-info/impure.json | 1 + src/libstore/binary-cache-store.cc | 8 +- src/libstore/binary-cache-store.hh | 9 +- src/libstore/build/derivation-goal.cc | 3 +- src/libstore/build/derivation-goal.hh | 3 + src/libstore/daemon.cc | 18 +++- src/libstore/derivations.cc | 18 +++- src/libstore/derivations.hh | 4 +- src/libstore/dummy-store.cc | 3 +- src/libstore/legacy-ssh-store.hh | 11 ++- src/libstore/local-store.cc | 22 ++++- src/libstore/local-store.hh | 3 +- src/libstore/meson.build | 2 + src/libstore/nar-info.cc | 7 ++ src/libstore/path-info.cc | 6 ++ src/libstore/path-info.hh | 7 ++ src/libstore/provenance.cc | 99 +++++++++++++++++++ src/libstore/provenance.hh | 97 ++++++++++++++++++ src/libstore/remote-store.cc | 26 +++-- src/libstore/remote-store.hh | 6 +- src/libstore/serve-protocol.hh | 4 + src/libstore/ssh-store.cc | 3 + src/libstore/store-api.cc | 46 ++++++--- src/libstore/store-api.hh | 15 ++- .../unix/build/local-derivation-goal.cc | 16 ++- src/libstore/worker-protocol-connection.cc | 2 +- src/libstore/worker-protocol-connection.hh | 2 + src/libstore/worker-protocol.cc | 9 +- src/libstore/worker-protocol.hh | 6 +- 38 files changed, 464 insertions(+), 57 deletions(-) create mode 100644 src/libstore/provenance.cc create mode 100644 src/libstore/provenance.hh diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 6c9ee674808..e8b46b9759a 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -17,6 +17,7 @@ #include "url.hh" #include "registry.hh" #include "build-result.hh" +#include "provenance.hh" #include #include @@ -81,6 +82,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto attrPath = attr->getAttrPathStr(); + auto lockedRef = getLockedFlake()->flake.lockedRef; + + state->setRootProvenance(std::make_shared( + Provenance::ProvFlake { + .flake = std::make_shared(fetchers::attrsToJSON(lockedRef.input.attrs)), + .flakeOutput = attrPath, + })); + if (!attr->isDerivation()) { // FIXME: use eval cache? @@ -147,7 +156,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() }, ExtraPathInfoFlake::Flake { .originalRef = flakeRef, - .lockedRef = getLockedFlake()->flake.lockedRef, + .lockedRef = lockedRef, }), }}; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f17753415fc..fc60697c374 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -20,6 +20,7 @@ #include "fetch-to-store.hh" #include "tarball.hh" #include "parser-tab.hh" +#include "provenance.hh" #include #include @@ -2364,7 +2365,8 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat path.baseName(), ContentAddressMethod::Raw::NixArchive, nullptr, - repair); + repair, + getRootProvenance()); allowPath(dstPath); srcToStore.lock()->try_emplace(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); @@ -3163,4 +3165,15 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } +std::shared_ptr EvalState::getRootProvenance() +{ + return rootProvenance; +} + + +void EvalState::setRootProvenance(std::shared_ptr provenance) +{ + rootProvenance = provenance; +} + } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f7ed6be83af..e665b4865d9 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -37,6 +37,7 @@ class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; struct MemorySourceAccessor; +struct Provenance; namespace eval_cache { class EvalCache; } @@ -863,6 +864,15 @@ private: friend struct Value; friend class ListBuilder; + + // FIXME: how to handle this in the multi-threaded evaluator? + std::shared_ptr rootProvenance; + +public: + + std::shared_ptr getRootProvenance(); + + void setRootProvenance(std::shared_ptr provenance); }; struct DebugTraceStacker { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 203d109324f..cad502826f9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1519,7 +1519,7 @@ static void derivationStrictInternal( } /* Write the resulting term into the Nix store directory. */ - auto drvPath = writeDerivation(*state.store, drv, state.repair); + auto drvPath = writeDerivation(*state.store, drv, state.repair, false, state.getRootProvenance()); auto drvPathS = state.store->printStorePath(drvPath); printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); @@ -2320,7 +2320,15 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val }) : ({ StringSource s { contents }; - state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair); + state.store->addToStoreFromDump( + s, + name, + FileSerialisationMethod::Flat, + ContentAddressMethod::Raw::Text, + HashAlgorithm::SHA256, + refs, + state.repair, + state.getRootProvenance()); }); /* Note: we don't need to add `context' to the context of the @@ -2480,7 +2488,8 @@ static void addPath( name, method, filter.get(), - state.repair); + state.repair, + state.getRootProvenance()); if (expectedHash && expectedStorePath != dstPath) state.error( "store path mismatch in (possibly filtered) path added from '%s'", diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 65aa72a6c36..6f7ea303222 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -11,7 +11,8 @@ StorePath fetchToStore( std::string_view name, ContentAddressMethod method, PathFilter * filter, - RepairFlag repair) + RepairFlag repair, + std::shared_ptr provenance) { // FIXME: add an optimisation for the case where the accessor is // a `PosixSourceAccessor` pointing to a store path. @@ -42,7 +43,7 @@ StorePath fetchToStore( ? store.computeStorePath( name, path, method, HashAlgorithm::SHA256, {}, filter2).first : store.addToStore( - name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); + name, path, method, HashAlgorithm::SHA256, {}, filter2, repair, provenance); if (cacheKey && mode == FetchMode::Copy) fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index c762629f3cb..7e5c48a06bb 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -20,6 +20,7 @@ StorePath fetchToStore( std::string_view name = "source", ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, PathFilter * filter = nullptr, - RepairFlag repair = NoRepair); + RepairFlag repair = NoRepair, + std::shared_ptr provenance = nullptr); } diff --git a/src/libstore-test-support/tests/protocol.hh b/src/libstore-test-support/tests/protocol.hh index 3f6799d1ccb..57526911fe4 100644 --- a/src/libstore-test-support/tests/protocol.hh +++ b/src/libstore-test-support/tests/protocol.hh @@ -32,11 +32,13 @@ public: CharacterizationTest::readTest(testStem, [&](const auto & encoded) { T got = ({ StringSource from { encoded }; + std::set features; Proto::template Serialise::read( *LibStoreTest::store, typename Proto::ReadConn { .from = from, .version = version, + .features = features, }); }); @@ -52,11 +54,13 @@ public: { CharacterizationTest::writeTest(testStem, [&]() { StringSink to; + std::set features; Proto::template Serialise::write( *LibStoreTest::store, typename Proto::WriteConn { .to = to, .version = version, + .features = features, }, decoded); return std::move(to.s); diff --git a/src/libstore-tests/data/nar-info/impure.json b/src/libstore-tests/data/nar-info/impure.json index bb9791a6ace..63f2dcfc4a4 100644 --- a/src/libstore-tests/data/nar-info/impure.json +++ b/src/libstore-tests/data/nar-info/impure.json @@ -6,6 +6,7 @@ "downloadSize": 4029176, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, + "provenance": null, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" diff --git a/src/libstore-tests/data/path-info/empty_impure.json b/src/libstore-tests/data/path-info/empty_impure.json index be982dcef85..fbaafd25d3b 100644 --- a/src/libstore-tests/data/path-info/empty_impure.json +++ b/src/libstore-tests/data/path-info/empty_impure.json @@ -3,6 +3,7 @@ "deriver": null, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 0, + "provenance": null, "references": [], "registrationTime": null, "signatures": [], diff --git a/src/libstore-tests/data/path-info/impure.json b/src/libstore-tests/data/path-info/impure.json index 0c452cc4930..ffc22406d43 100644 --- a/src/libstore-tests/data/path-info/impure.json +++ b/src/libstore-tests/data/path-info/impure.json @@ -3,6 +3,7 @@ "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, + "provenance": null, "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e8c8892b337..4be055d11dc 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -302,7 +302,8 @@ StorePath BinaryCacheStore::addToStoreFromDump( ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) + RepairFlag repair, + std::shared_ptr provenance) { std::optional caHash; std::string nar; @@ -364,6 +365,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( nar.first, }; info.narSize = nar.second; + info.provenance = provenance; return info; })->path; } @@ -448,7 +450,8 @@ StorePath BinaryCacheStore::addToStore( HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) + RepairFlag repair, + std::shared_ptr provenance) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default @@ -474,6 +477,7 @@ StorePath BinaryCacheStore::addToStore( nar.first, }; info.narSize = nar.second; + info.provenance = provenance; return info; })->path; } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 695bc925277..f300c05cc0e 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -68,6 +68,9 @@ protected: public: + bool uriIsUsefulProvenance() override + { return true; } + virtual bool fileExists(const std::string & path) = 0; virtual void upsertFile(const std::string & path, @@ -129,7 +132,8 @@ public: ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) override; + RepairFlag repair, + std::shared_ptr provenance) override; StorePath addToStore( std::string_view name, @@ -138,7 +142,8 @@ public: HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) override; + RepairFlag repair, + std::shared_ptr provenance) override; void registerDrvOutput(const Realisation & info) override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 34ed16a3819..8d9ba11aaa7 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -172,7 +172,7 @@ Goal::Co DerivationGoal::loadDerivation() things being garbage collected while we're busy. */ worker.evalStore.addTempRoot(drvPath); - /* Get the derivation. It is probably in the eval store, but it might be inthe main store: + /* Get the derivation. It is probably in the eval store, but it might be in the main store: - Resolved derivation are resolved against main store realisations, and so must be stored there. @@ -181,6 +181,7 @@ Goal::Co DerivationGoal::loadDerivation() for (auto * drvStore : { &worker.evalStore, &worker.store }) { if (drvStore->isValidPath(drvPath)) { drv = std::make_unique(drvStore->readDerivation(drvPath)); + drvProvenance = drvStore->queryPathInfo(drvPath)->provenance; break; } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index ad3d9ca2acf..d93fd004c94 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -67,6 +67,9 @@ struct DerivationGoal : public Goal /** The path of the derivation. */ StorePath drvPath; + /** The provenance of the derivation, if any. */ + std::shared_ptr drvProvenance; + /** * The goal for the corresponding resolved derivation */ diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index b921dbe2de8..91995492cc1 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -15,6 +15,7 @@ #include "derivations.hh" #include "args.hh" #include "git.hh" +#include "provenance.hh" #ifndef _WIN32 // TODO need graceful async exit support on Windows? # include "monitor-fd.hh" @@ -22,6 +23,8 @@ #include +#include + namespace nix::daemon { Sink & operator << (Sink & sink, const Logger::Fields & fields) @@ -398,6 +401,11 @@ static void performOp(TunnelLogger * logger, ref store, bool repairBool; conn.from >> repairBool; auto repair = RepairFlag{repairBool}; + std::shared_ptr provenance; + if (conn.features.contains(WorkerProto::featureProvenance)) + provenance = + nlohmann::json::parse(readString(conn.from)) + .template get>(); logger->startWork(); auto pathInfo = [&]() { @@ -423,7 +431,15 @@ static void performOp(TunnelLogger * logger, ref store, assert(false); } // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store. - auto path = store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair); + auto path = store->addToStoreFromDump( + source, + name, + dumpMethod, + contentAddressMethod, + hashAlgo, + refs, + repair, + provenance); return store->queryPathInfo(path); }(); logger->stopWork(); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 9b6f6785291..be8421bc9ff 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -135,8 +135,12 @@ bool BasicDerivation::isBuiltin() const } -StorePath writeDerivation(Store & store, - const Derivation & drv, RepairFlag repair, bool readOnly) +StorePath writeDerivation( + Store & store, + const Derivation & drv, + RepairFlag repair, + bool readOnly, + std::shared_ptr provenance) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs.map) @@ -153,7 +157,15 @@ StorePath writeDerivation(Store & store, }) : ({ StringSource s { contents }; - store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, repair); + store.addToStoreFromDump( + s, + suffix, + FileSerialisationMethod::Flat, + ContentAddressMethod::Raw::Text, + HashAlgorithm::SHA256, + references, + repair, + provenance); }); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 58e5328a5eb..6e6668ae411 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -16,6 +16,7 @@ namespace nix { struct StoreDirConfig; +struct Provenance; /* Abstract syntax of derivations. */ @@ -395,7 +396,8 @@ class Store; StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair = NoRepair, - bool readOnly = false); + bool readOnly = false, + std::shared_ptr provenance = nullptr); /** * Read a derivation from a file. diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c1e871e9384..7a612012561 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -72,7 +72,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) override + RepairFlag repair = NoRepair, + std::shared_ptr provenance = nullptr) override { unsupported("addToStore"); } void narFromPath(const StorePath & path, Sink & sink) override diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index b541455b4e5..9d084ef29ab 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -57,6 +57,9 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor std::string getUri() override; + bool uriIsUsefulProvenance() override + { return true; } + void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept override; @@ -75,17 +78,19 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) override + RepairFlag repair, + std::shared_ptr provenance) override { unsupported("addToStore"); } - virtual StorePath addToStoreFromDump( + StorePath addToStoreFromDump( Source & dump, std::string_view name, FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) override + RepairFlag repair = NoRepair, + std::shared_ptr provenance = nullptr) override { unsupported("addToStore"); } public: diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f708bd1b008..c41bf34e446 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -17,6 +17,7 @@ #include "posix-source-accessor.hh" #include "keys.hh" #include "users.hh" +#include "provenance.hh" #include #include @@ -305,13 +306,13 @@ LocalStore::LocalStore( /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create(state->db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca, provenance) values (?, ?, ?, ?, ?, ?, ?, ?, ?);"); state->stmts->UpdatePathInfo.create(state->db, "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); state->stmts->AddReference.create(state->db, "insert or replace into Refs (referrer, reference) values (?, ?);"); state->stmts->QueryPathInfo.create(state->db, - "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca, provenance from ValidPaths where path = ?;"); state->stmts->QueryReferences.create(state->db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); state->stmts->QueryReferrers.create(state->db, @@ -562,6 +563,10 @@ void LocalStore::upgradeDBSchema(State & state) "20220326-ca-derivations", #include "ca-specific-schema.sql.gen.hh" ); + + doUpgrade( + "202410124-provenance", + "alter table ValidPaths add column provenance text"); } @@ -679,6 +684,10 @@ uint64_t LocalStore::addValidPath(State & state, (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) + (info.provenance + ? nlohmann::json(*info.provenance).dump() + : "", + (bool) info.provenance) .exec(); uint64_t id = state.db.getLastInsertedRowId(); @@ -770,6 +779,10 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s while (useQueryReferences.next()) info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + auto prov = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 8); + if (prov) + info->provenance = std::make_shared(nlohmann::json::parse(prov).template get()); + return info; } @@ -1162,7 +1175,8 @@ StorePath LocalStore::addToStoreFromDump( ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) + RepairFlag repair, + std::shared_ptr provenance) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1262,7 +1276,6 @@ StorePath LocalStore::addToStoreFromDump( PathLocks outputLock({realPath}); if (repair || !isValidPath(dstPath)) { - deletePath(realPath); autoGC(); @@ -1311,6 +1324,7 @@ StorePath LocalStore::addToStoreFromDump( narHash.first }; info.narSize = narHash.second; + info.provenance = provenance; registerValidPath(info); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 83154d65193..5516034e297 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -191,7 +191,8 @@ public: ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) override; + RepairFlag repair, + std::shared_ptr provenance = nullptr) override; void addTempRoot(const StorePath & path) override; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 6a6aabf97aa..6a192e90389 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -220,6 +220,7 @@ sources = files( 'pathlocks.cc', 'posix-fs-canonicalise.cc', 'profiles.cc', + 'provenance.cc', 'realisation.cc', 'remote-fs-accessor.cc', 'remote-store.cc', @@ -289,6 +290,7 @@ headers = [config_h] + files( 'pathlocks.hh', 'posix-fs-canonicalise.hh', 'profiles.hh', + 'provenance.hh', 'realisation.hh', 'remote-fs-accessor.hh', 'remote-store-connection.hh', diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 8b255706072..b8b3b9af26d 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "strings.hh" #include "json-utils.hh" +#include "provenance.hh" namespace nix { @@ -81,6 +82,9 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & // FIXME: allow blank ca or require skipping field? ca = ContentAddress::parseOpt(value); } + else if (name == "Provenance") { + provenance = std::make_shared(nlohmann::json::parse(value).template get()); + } pos = eol + 1; line += 1; @@ -124,6 +128,9 @@ std::string NarInfo::to_string(const Store & store) const if (ca) res += "CA: " + renderContentAddress(*ca) + "\n"; + if (provenance) + res += "Provenance: " + nlohmann::json(*provenance).dump() + "\n"; + return res; } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 6e87e60f446..f140c9bb57d 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -5,6 +5,7 @@ #include "json-utils.hh" #include "comparator.hh" #include "strings.hh" +#include "provenance.hh" namespace nix { @@ -177,6 +178,8 @@ nlohmann::json UnkeyedValidPathInfo::toJSON( auto & sigsObj = jsonObject["signatures"] = json::array(); for (auto & sig : sigs) sigsObj.push_back(sig); + + jsonObject["provenance"] = provenance ? nlohmann::json(*provenance) : nlohmann::json(); } return jsonObject; @@ -224,6 +227,9 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( if (json.contains("signatures")) res.sigs = getStringSet(valueAt(json, "signatures")); + auto prov = json.find("provenance"); + res.provenance = prov == json.end() || prov->second.is_null() ? nullptr : std::make_shared(prov->second.get()); + return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 71f1476a672..7f80ad91326 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -13,6 +13,7 @@ namespace nix { class Store; +struct Provenance; struct SubstitutablePathInfo @@ -101,6 +102,12 @@ struct UnkeyedValidPathInfo */ std::optional ca; + /** + * The provenance of this store path, i.e. a link back to the Nix + * expression used to create it. + */ + std::shared_ptr provenance; + UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default; UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; diff --git a/src/libstore/provenance.cc b/src/libstore/provenance.cc new file mode 100644 index 00000000000..d42e74d641f --- /dev/null +++ b/src/libstore/provenance.cc @@ -0,0 +1,99 @@ +#include "provenance.hh" +#include "util.hh" + +#include + +namespace nix { + +std::string Provenance::type() const +{ + return std::visit( + overloaded{ + [&](const Provenance::ProvDerivation & p) -> std::string { return "derivation"; }, + [&](const Provenance::ProvCopied & p) -> std::string { return "copied"; }, + [&](const Provenance::ProvFlake & p) -> std::string { return "flake"; }, + }, + raw); +} + +void to_json(nlohmann::json & j, const std::shared_ptr & p) +{ + if (p) + to_json(j, *p); + else + j = nlohmann::json(); +} + +void to_json(nlohmann::json & j, const Provenance & p) +{ + std::visit( + overloaded{ + [&](const Provenance::ProvDerivation & p) { nlohmann::to_json(j, p); }, + [&](const Provenance::ProvCopied & p) { nlohmann::to_json(j, p); }, + [&](const Provenance::ProvFlake & p) { nlohmann::to_json(j, p); }, + }, + p.raw); + + j["type"] = p.type(); +} + +void to_json(nlohmann::json & j, const Provenance::ProvDerivation & p) +{ + j = nlohmann::json{ + {"drv", p.drvPath.to_string()}, {"provenance", nlohmann::json(p.provenance)}, {"output", p.output}}; +} + +void to_json(nlohmann::json & j, const Provenance::ProvCopied & p) +{ + j = nlohmann::json{ + {"from", p.from}, + {"provenance", nlohmann::json(p.provenance)}, + }; +} + +void to_json(nlohmann::json & j, const Provenance::ProvFlake & p) +{ + j = nlohmann::json{ + {"flake", *p.flake}, + {"output", p.flakeOutput}, + }; +} + +void from_json(const nlohmann::json & j, std::shared_ptr & p) +{ + p = j.is_null() ? nullptr : std::make_shared(j.get()); +} + +void from_json(const nlohmann::json & j, Provenance & p) +{ + auto type = j.at("type").get(); + + if (type == "flake") + p = {Provenance::ProvFlake{ + .flake = std::make_shared(j.at("flake")), // FIXME: validate + .flakeOutput = j.at("output").get(), + }}; + + else if (type == "derivation") { + auto prov = j.at("provenance"); + p = {Provenance::ProvDerivation{ + .drvPath = StorePath(j.at("drv").get()), + .output = j.at("output").get(), + .provenance = prov.get>(), + }}; + } + + else if (type == "copied") { + auto prov = j.at("provenance"); + p = {Provenance::ProvCopied{ + .from = j.at("from").get(), + .provenance = prov.get>(), + }}; + } + + else + // FIXME: pass this through as raw json? + throw Error("unsupported provenance type '%s'", type); +} + +} diff --git a/src/libstore/provenance.hh b/src/libstore/provenance.hh new file mode 100644 index 00000000000..ee8debfff39 --- /dev/null +++ b/src/libstore/provenance.hh @@ -0,0 +1,97 @@ +#pragma once + +#include "outputs-spec.hh" +#include "path.hh" + +#include "nlohmann/json_fwd.hpp" + +namespace nix { + +/** + * This struct describes the provenance of a store path, i.e. a link + * back to the source code from which the store path was originally + * built. + */ +struct Provenance +{ + /** + * Type that denotes a store path that was produced by a + * derivation. + */ + struct ProvDerivation + { + /** + * The derivation that built this path. + */ + StorePath drvPath; + + /** + * The output of the derivation that corresponds to this path. + */ + OutputName output; + + /** + * The nested provenance of the derivation. + */ + std::shared_ptr provenance; + + // FIXME: do we need anything extra for CA derivations? + }; + + /** + * Type that denotes a store path that was copied/substituted + * from another store. + */ + struct ProvCopied + { + /** + * Store URL (typically a binary cache) from which this store + * path was copied. + */ + std::string from; + + /** + * Provenance of the store path in the upstream store, if any. + */ + std::shared_ptr provenance; + }; + + /** + * Type that denotes a store path (typically a .drv file or + * derivation input source) that was produced by the evaluation of + * a flake. + */ + struct ProvFlake + { + std::shared_ptr flake; // FIXME: change to Attrs + std::string flakeOutput; + }; + + using Raw = std::variant; + + Raw raw; + + Provenance(Raw raw) + : raw(std::move(raw)) + { + } + + // FIXME: ugly, nlohmann::json wants a default constructor. + Provenance() + : raw(ProvFlake{}) + { + } + + std::string type() const; +}; + +void to_json(nlohmann::json & j, const std::shared_ptr & p); +void to_json(nlohmann::json & j, const Provenance & p); +void to_json(nlohmann::json & j, const Provenance::ProvDerivation & p); +void to_json(nlohmann::json & j, const Provenance::ProvCopied & p); +void to_json(nlohmann::json & j, const Provenance::ProvFlake & p); + +void from_json(const nlohmann::json & j, std::shared_ptr & p); +void from_json(const nlohmann::json & j, Provenance & p); + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 69bbc64fca3..466e2f57d8e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -18,6 +18,7 @@ #include "callback.hh" #include "filetransfer.hh" #include "signals.hh" +#include "provenance.hh" #include @@ -364,12 +365,13 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & ref RemoteStore::addCAToStore( - Source & dump, - std::string_view name, - ContentAddressMethod caMethod, - HashAlgorithm hashAlgo, - const StorePathSet & references, - RepairFlag repair) + Source & dump, + std::string_view name, + ContentAddressMethod caMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair, + std::shared_ptr provenance) { std::optional conn_(getConnection()); auto & conn = *conn_; @@ -382,6 +384,8 @@ ref RemoteStore::addCAToStore( << caMethod.renderWithAlgo(hashAlgo); WorkerProto::write(*this, *conn, references); conn->to << repair; + if (conn->features.contains(WorkerProto::featureProvenance)) + conn->to << nlohmann::json(provenance).dump(); // The dump source may invoke the store, so we need to make some room. connections->incCapacity(); @@ -463,7 +467,8 @@ StorePath RemoteStore::addToStoreFromDump( ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) + RepairFlag repair, + std::shared_ptr provenance) { FileSerialisationMethod fsm; switch (hashMethod.getFileIngestionMethod()) { @@ -482,13 +487,16 @@ StorePath RemoteStore::addToStoreFromDump( } if (fsm != dumpMethod) unsupported("RemoteStore::addToStoreFromDump doesn't support this `dumpMethod` `hashMethod` combination"); - return addCAToStore(dump, name, hashMethod, hashAlgo, references, repair)->path; + return addCAToStore(dump, name, hashMethod, hashAlgo, references, repair, provenance)->path; } void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { + if (info.provenance) + throw UnimplementedError("RemoteStore::addToStore() with provenance"); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->protoVersion) < 18) { @@ -539,6 +547,7 @@ void RemoteStore::addMultipleToStore( RepairFlag repair, CheckSigsFlag checkSigs) { + std::set features; auto source = sinkToSource([&](Sink & sink) { sink << pathsToCopy.size(); for (auto & [pathInfo, pathSource] : pathsToCopy) { @@ -546,6 +555,7 @@ void RemoteStore::addMultipleToStore( WorkerProto::WriteConn { .to = sink, .version = 16, + .features = features, // FIXME }, pathInfo); pathSource->drainInto(sink); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4e18962683d..50ae8521655 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -79,7 +79,8 @@ public: ContentAddressMethod caMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair); + RepairFlag repair, + std::shared_ptr provenance); /** * Add a content-addressable store path. `dump` will be drained. @@ -91,7 +92,8 @@ public: ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) override; + RepairFlag repair = NoRepair, + std::shared_ptr provenance = nullptr) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 8c112bb74c7..3e87c42d54b 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -41,6 +41,8 @@ struct ServeProto */ using Version = unsigned int; + using Feature = std::string; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -48,6 +50,7 @@ struct ServeProto struct ReadConn { Source & from; Version version; + std::set features; // unused }; /** @@ -57,6 +60,7 @@ struct ServeProto struct WriteConn { Sink & to; Version version; + std::set features; // unused }; /** diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 954a9746774..b0a99ed5284 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -52,6 +52,9 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore return *uriSchemes().begin() + "://" + host; } + bool uriIsUsefulProvenance() override + { return true; } + // FIXME extend daemon protocol, move implementation to RemoteStore std::optional getBuildLogExact(const StorePath & path) override { unsupported("getBuildLogExact"); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 10577fa2a32..0209f368fb9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -18,6 +18,8 @@ #include "worker-protocol.hh" #include "signals.hh" #include "users.hh" +#include "provenance.hh" +#include "local-fs-store.hh" #include #include @@ -195,7 +197,8 @@ StorePath Store::addToStore( HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) + RepairFlag repair, + std::shared_ptr provenance) { FileSerialisationMethod fsm; switch (method.getFileIngestionMethod()) { @@ -213,7 +216,7 @@ StorePath Store::addToStore( std::optional storePath; auto sink = sourceToSink([&](Source & source) { LengthSource lengthSource(source); - storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); + storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair, provenance); if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); }); @@ -306,6 +309,7 @@ void Store::addMultipleToStore( RepairFlag repair, CheckSigsFlag checkSigs) { + std::set features; auto expected = readNum(source); for (uint64_t i = 0; i < expected; ++i) { // FIXME we should not be using the worker protocol here, let @@ -314,6 +318,7 @@ void Store::addMultipleToStore( WorkerProto::ReadConn { .from = source, .version = 16, + .features = features, // FIXME }); info.ultimate = false; addToStore(info, source, repair, checkSigs); @@ -953,6 +958,25 @@ static std::string makeCopyPathMessage( } +/** + * Wrap upstream provenance in a "copied" provenance record to record + * where the path was copied from. But uninformative origins like + * LocalStore are omitted. + */ +static std::shared_ptr addCopiedProvenance( + std::shared_ptr prov, + Store & srcStore) +{ + if (!srcStore.uriIsUsefulProvenance()) + return prov; + return std::make_shared( + Provenance::ProvCopied { + .from = srcStore.getUri(), + .provenance = prov, + }); +} + + void copyStorePath( Store & srcStore, Store & dstStore, @@ -973,26 +997,23 @@ void copyStorePath( {storePathS, srcUri, dstUri}); PushActivity pact(act.id); - auto info = srcStore.queryPathInfo(storePath); + auto srcInfo = srcStore.queryPathInfo(storePath); + auto info = make_ref(*srcInfo); uint64_t total = 0; // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { - auto info2 = make_ref(*info); - info2->path = dstStore.makeFixedOutputPathFromCA( + info->path = dstStore.makeFixedOutputPathFromCA( info->path.name(), info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) - assert(info->path == info2->path); - info = info2; + assert(info->path == srcInfo->path); } - if (info->ultimate) { - auto info2 = make_ref(*info); - info2->ultimate = false; - info = info2; - } + info->ultimate = false; + + info->provenance = addCopiedProvenance(info->provenance, srcStore); auto source = sinkToSource([&](Sink & sink) { LambdaSink progressSink([&](std::string_view data) { @@ -1119,6 +1140,7 @@ std::map copyPaths( ValidPathInfo infoForDst = *info; infoForDst.path = storePathForDst; + infoForDst.provenance = addCopiedProvenance(info->provenance, srcStore); auto source = sinkToSource([&](Sink & sink) { // We can reasonably assume that the copy will happen whenever we diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8288cfdf01f..f87471f32ec 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -75,6 +75,8 @@ struct SourceAccessor; class NarInfoDiskCache; class Store; +struct Provenance; + typedef std::map OutputPathMap; @@ -222,6 +224,13 @@ public: */ virtual std::string getUri() = 0; + /** + * Whether, when copying *from* this store, a "copied" provenance + * record should be added. + */ + virtual bool uriIsUsefulProvenance() + { return false; } + /** * Follow symlinks until we end up with a path in the Nix store. */ @@ -445,7 +454,8 @@ public: HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), PathFilter & filter = defaultPathFilter, - RepairFlag repair = NoRepair); + RepairFlag repair = NoRepair, + std::shared_ptr provenance = nullptr); /** * Copy the contents of a path to the store and register the @@ -484,7 +494,8 @@ public: ContentAddressMethod hashMethod = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), - RepairFlag repair = NoRepair) = 0; + RepairFlag repair = NoRepair, + std::shared_ptr provenance = nullptr) = 0; /** * Add a mapping indicating that `deriver!outputName` maps to the output path diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index b4685b3a77e..8dee2c2d6b6 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -19,6 +19,7 @@ #include "unix-domain-socket.hh" #include "posix-fs-canonicalise.hh" #include "posix-source-accessor.hh" +#include "provenance.hh" #include #include @@ -1381,7 +1382,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In HashAlgorithm hashAlgo, const StorePathSet & references, PathFilter & filter, - RepairFlag repair) override + RepairFlag repair, + std::shared_ptr provenance) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1398,9 +1400,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, - RepairFlag repair) override + RepairFlag repair, + std::shared_ptr provenance) override { - auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair); + auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair, provenance); goal.addDependency(path); return path; } @@ -2787,6 +2790,13 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() newInfo.deriver = drvPath; newInfo.ultimate = true; + if (drvProvenance) + newInfo.provenance = std::make_shared( + Provenance::ProvDerivation { + .drvPath = drvPath, + .output = outputName, + .provenance = drvProvenance, + }); localStore.signPathInfo(newInfo); finish(newInfo.path); diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 6585df4be62..3d0471b5e7d 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,7 +5,7 @@ namespace nix { -const std::set WorkerProto::allFeatures{}; +const std::set WorkerProto::allFeatures{{WorkerProto::featureProvenance}}; WorkerProto::BasicClientConnection::~BasicClientConnection() { diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh index 9665067ddef..94250001898 100644 --- a/src/libstore/worker-protocol-connection.hh +++ b/src/libstore/worker-protocol-connection.hh @@ -41,6 +41,7 @@ struct WorkerProto::BasicConnection return WorkerProto::ReadConn{ .from = from, .version = protoVersion, + .features = features, }; } @@ -57,6 +58,7 @@ struct WorkerProto::BasicConnection return WorkerProto::WriteConn{ .to = to, .version = protoVersion, + .features = features, }; } }; diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index f06fb2893c7..893eebbcf92 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -6,6 +6,7 @@ #include "worker-protocol-impl.hh" #include "archive.hh" #include "path-info.hh" +#include "provenance.hh" #include #include @@ -232,6 +233,10 @@ UnkeyedValidPathInfo WorkerProto::Serialise::read(const St info.sigs = readStrings(conn.from); info.ca = ContentAddress::parseOpt(readString(conn.from)); } + if (conn.features.contains(WorkerProto::featureProvenance)) + info.provenance = + nlohmann::json::parse(readString(conn.from)) + .template get>(); return info; } @@ -248,6 +253,8 @@ void WorkerProto::Serialise::write(const StoreDirConfig & << pathInfo.sigs << renderContentAddress(pathInfo.ca); } + if (conn.features.contains(WorkerProto::featureProvenance)) + conn.to << nlohmann::json(pathInfo.provenance).dump(); } @@ -260,7 +267,7 @@ WorkerProto::ClientHandshakeInfo WorkerProto::Serialise= 35) { - res.remoteTrustsUs = WorkerProto::Serialise>::read(store, conn); + res.remoteTrustsUs = WorkerProto::Serialise>::read(store, conn); } else { // We don't know the answer; protocol to old. res.remoteTrustsUs = std::nullopt; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c356fa1bf37..b997b45e0a8 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -61,6 +61,8 @@ struct WorkerProto */ using Version = unsigned int; + using Feature = std::string; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -68,6 +70,7 @@ struct WorkerProto struct ReadConn { Source & from; Version version; + std::set & features; }; /** @@ -77,6 +80,7 @@ struct WorkerProto struct WriteConn { Sink & to; Version version; + std::set & features; }; /** @@ -134,7 +138,7 @@ struct WorkerProto WorkerProto::Serialise::write(store, conn, t); } - using Feature = std::string; + static constexpr std::string featureProvenance{"provenance"}; static const std::set allFeatures; }; From 9f6746af3ba2370afb373a7ba61c25100cb1c235 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 27 Oct 2024 12:04:44 +0100 Subject: [PATCH 2/3] Record provenance of source trees --- src/libfetchers/fetchers.cc | 16 +++++++++++++++- src/libstore/provenance.cc | 16 ++++++++++++++++ src/libstore/provenance.hh | 14 +++++++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index b07e8cb6ee7..db6c4a7005c 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -3,6 +3,7 @@ #include "source-path.hh" #include "fetch-to-store.hh" #include "json-utils.hh" +#include "provenance.hh" #include @@ -174,7 +175,20 @@ std::pair Input::fetchToStore(ref store) const try { auto [accessor, final] = getAccessorUnchecked(store); - auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, final.getName()); + auto storePath = nix::fetchToStore( + *store, + SourcePath(accessor), + FetchMode::Copy, + final.getName(), + ContentAddressMethod::Raw::NixArchive, + nullptr, + NoRepair, + std::make_shared( + Provenance::ProvSourcePath { + .tree = std::make_shared(fetchers::attrsToJSON(final.attrs)), + .path = CanonPath::root, + } + )); auto narHash = store->queryPathInfo(storePath)->narHash; final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); diff --git a/src/libstore/provenance.cc b/src/libstore/provenance.cc index d42e74d641f..8e39527bac8 100644 --- a/src/libstore/provenance.cc +++ b/src/libstore/provenance.cc @@ -11,6 +11,7 @@ std::string Provenance::type() const overloaded{ [&](const Provenance::ProvDerivation & p) -> std::string { return "derivation"; }, [&](const Provenance::ProvCopied & p) -> std::string { return "copied"; }, + [&](const Provenance::ProvSourcePath & p) -> std::string { return "source"; }, [&](const Provenance::ProvFlake & p) -> std::string { return "flake"; }, }, raw); @@ -30,6 +31,7 @@ void to_json(nlohmann::json & j, const Provenance & p) overloaded{ [&](const Provenance::ProvDerivation & p) { nlohmann::to_json(j, p); }, [&](const Provenance::ProvCopied & p) { nlohmann::to_json(j, p); }, + [&](const Provenance::ProvSourcePath & p) { nlohmann::to_json(j, p); }, [&](const Provenance::ProvFlake & p) { nlohmann::to_json(j, p); }, }, p.raw); @@ -51,6 +53,14 @@ void to_json(nlohmann::json & j, const Provenance::ProvCopied & p) }; } +void to_json(nlohmann::json & j, const Provenance::ProvSourcePath & p) +{ + j = nlohmann::json{ + {"tree", *p.tree}, + {"path", p.path.abs()}, + }; +} + void to_json(nlohmann::json & j, const Provenance::ProvFlake & p) { j = nlohmann::json{ @@ -74,6 +84,12 @@ void from_json(const nlohmann::json & j, Provenance & p) .flakeOutput = j.at("output").get(), }}; + else if (type == "source") + p = {Provenance::ProvSourcePath{ + .tree = std::make_shared(j.at("tree")), // FIXME: validate + .path = CanonPath(j.at("path").get()), + }}; + else if (type == "derivation") { auto prov = j.at("provenance"); p = {Provenance::ProvDerivation{ diff --git a/src/libstore/provenance.hh b/src/libstore/provenance.hh index ee8debfff39..0d50457b884 100644 --- a/src/libstore/provenance.hh +++ b/src/libstore/provenance.hh @@ -2,6 +2,7 @@ #include "outputs-spec.hh" #include "path.hh" +#include "canon-path.hh" #include "nlohmann/json_fwd.hpp" @@ -56,6 +57,16 @@ struct Provenance std::shared_ptr provenance; }; + /** + * Type that denotes a store path that was produced by copying a + * path inside a source tree. + */ + struct ProvSourcePath + { + std::shared_ptr tree; // FIXME: change to Attrs + CanonPath path; + }; + /** * Type that denotes a store path (typically a .drv file or * derivation input source) that was produced by the evaluation of @@ -67,7 +78,7 @@ struct Provenance std::string flakeOutput; }; - using Raw = std::variant; + using Raw = std::variant; Raw raw; @@ -89,6 +100,7 @@ void to_json(nlohmann::json & j, const std::shared_ptr & p); void to_json(nlohmann::json & j, const Provenance & p); void to_json(nlohmann::json & j, const Provenance::ProvDerivation & p); void to_json(nlohmann::json & j, const Provenance::ProvCopied & p); +void to_json(nlohmann::json & j, const Provenance::ProvSourcePath & p); void to_json(nlohmann::json & j, const Provenance::ProvFlake & p); void from_json(const nlohmann::json & j, std::shared_ptr & p); From 78f7ac43127d662179119c880a6ccc670af54141 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 21:50:45 +0100 Subject: [PATCH 3/3] Fix migration name --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c41bf34e446..d77d4ed6d77 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -565,7 +565,7 @@ void LocalStore::upgradeDBSchema(State & state) ); doUpgrade( - "202410124-provenance", + "20241024-provenance", "alter table ValidPaths add column provenance text"); }