Store path provenance tracking

src/libcmd/installable-flake.cc

@@ -17,6 +17,7 @@
 #include "url.hh"
 #include "registry.hh"
 #include "build-result.hh"
+#include "provenance.hh"
 
 #include <regex>
 #include <queue>
@@ -81,6 +82,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     auto attrPath = attr->getAttrPathStr();
 
+    auto lockedRef = getLockedFlake()->flake.lockedRef;
+
+    state->setRootProvenance(std::make_shared<Provenance>(
+            Provenance::ProvFlake {
+                .flake = std::make_shared<nlohmann::json>(fetchers::attrsToJSON(lockedRef.input.attrs)),
+                .flakeOutput = attrPath,
+            }));
+
     if (!attr->isDerivation()) {
 
         // FIXME: use eval cache?
@@ -147,7 +156,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
             },
             ExtraPathInfoFlake::Flake {
                 .originalRef = flakeRef,
-                .lockedRef = getLockedFlake()->flake.lockedRef,
+                .lockedRef = lockedRef,
             }),
     }};
 }

src/libexpr/eval.cc

@@ -20,6 +20,7 @@
 #include "fetch-to-store.hh"
 #include "tarball.hh"
 #include "parser-tab.hh"
+#include "provenance.hh"
 
 #include <algorithm>
 #include <iostream>
@@ -2364,7 +2365,8 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
                 path.baseName(),
                 ContentAddressMethod::Raw::NixArchive,
                 nullptr,
-                repair);
+                repair,
+                getRootProvenance());
             allowPath(dstPath);
             srcToStore.lock()->try_emplace(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@@ -3163,4 +3165,15 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
+std::shared_ptr<const Provenance> EvalState::getRootProvenance()
+{
+    return rootProvenance;
+}
+
+
+void EvalState::setRootProvenance(std::shared_ptr<const Provenance> provenance)
+{
+    rootProvenance = provenance;
+}
+
 }

src/libexpr/eval.hh

@@ -37,6 +37,7 @@ class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
 struct MemorySourceAccessor;
+struct Provenance;
 namespace eval_cache {
     class EvalCache;
 }
@@ -863,6 +864,15 @@ private:
 
     friend struct Value;
     friend class ListBuilder;
+
+    // FIXME: how to handle this in the multi-threaded evaluator?
+    std::shared_ptr<const Provenance> rootProvenance;
+
+public:
+
+    std::shared_ptr<const Provenance> getRootProvenance();
+
+    void setRootProvenance(std::shared_ptr<const Provenance> provenance);
 };
 
 struct DebugTraceStacker {

src/libexpr/primops.cc

@@ -1519,7 +1519,7 @@ static void derivationStrictInternal(
     }
 
     /* Write the resulting term into the Nix store directory. */
-    auto drvPath = writeDerivation(*state.store, drv, state.repair);
+    auto drvPath = writeDerivation(*state.store, drv, state.repair, false, state.getRootProvenance());
     auto drvPathS = state.store->printStorePath(drvPath);
 
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -2320,7 +2320,15 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
         })
         : ({
             StringSource s { contents };
-            state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair);
+            state.store->addToStoreFromDump(
+                s,
+                name,
+                FileSerialisationMethod::Flat,
+                ContentAddressMethod::Raw::Text,
+                HashAlgorithm::SHA256,
+                refs,
+                state.repair,
+                state.getRootProvenance());
         });
 
     /* Note: we don't need to add `context' to the context of the
@@ -2480,7 +2488,8 @@ static void addPath(
                 name,
                 method,
                 filter.get(),
-                state.repair);
+                state.repair,
+                state.getRootProvenance());
             if (expectedHash && expectedStorePath != dstPath)
                 state.error<EvalError>(
                     "store path mismatch in (possibly filtered) path added from '%s'",

src/libfetchers/fetch-to-store.cc

@@ -11,7 +11,8 @@ StorePath fetchToStore(
     std::string_view name,
     ContentAddressMethod method,
     PathFilter * filter,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::shared_ptr<const Provenance> provenance)
 {
     // FIXME: add an optimisation for the case where the accessor is
     // a `PosixSourceAccessor` pointing to a store path.
@@ -42,7 +43,7 @@ StorePath fetchToStore(
         ? store.computeStorePath(
             name, path, method, HashAlgorithm::SHA256, {}, filter2).first
         : store.addToStore(
-            name, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
+            name, path, method, HashAlgorithm::SHA256, {}, filter2, repair, provenance);
 
     if (cacheKey && mode == FetchMode::Copy)
         fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);

src/libfetchers/fetch-to-store.hh

@@ -20,6 +20,7 @@ StorePath fetchToStore(
     std::string_view name = "source",
     ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
     PathFilter * filter = nullptr,
-    RepairFlag repair = NoRepair);
+    RepairFlag repair = NoRepair,
+    std::shared_ptr<const Provenance> provenance = nullptr);
 
 }

src/libfetchers/fetchers.cc

@@ -3,6 +3,7 @@
 #include "source-path.hh"
 #include "fetch-to-store.hh"
 #include "json-utils.hh"
+#include "provenance.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -174,7 +175,20 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
         try {
             auto [accessor, final] = getAccessorUnchecked(store);
 
-            auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, final.getName());
+            auto storePath = nix::fetchToStore(
+                *store,
+                SourcePath(accessor),
+                FetchMode::Copy,
+                final.getName(),
+                ContentAddressMethod::Raw::NixArchive,
+                nullptr,
+                NoRepair,
+                std::make_shared<Provenance>(
+                    Provenance::ProvSourcePath {
+                        .tree = std::make_shared<nlohmann::json>(fetchers::attrsToJSON(final.attrs)),
+                        .path = CanonPath::root,
+                    }
+                ));
 
             auto narHash = store->queryPathInfo(storePath)->narHash;
             final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));

src/libstore-test-support/tests/protocol.hh

@@ -32,11 +32,13 @@ public:
         CharacterizationTest::readTest(testStem, [&](const auto & encoded) {
             T got = ({
                 StringSource from { encoded };
+                std::set<std::string> features;
                 Proto::template Serialise<T>::read(
                     *LibStoreTest::store,
                     typename Proto::ReadConn {
                         .from = from,
                         .version = version,
+                        .features = features,
                     });
             });
 
@@ -52,11 +54,13 @@ public:
     {
         CharacterizationTest::writeTest(testStem, [&]() {
             StringSink to;
+            std::set<std::string> features;
             Proto::template Serialise<T>::write(
                 *LibStoreTest::store,
                 typename Proto::WriteConn {
                     .to = to,
                     .version = version,
+                    .features = features,
                 },
                 decoded);
             return std::move(to.s);

src/libstore-tests/data/nar-info/impure.json

@@ -6,6 +6,7 @@
   "downloadSize": 4029176,
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
+  "provenance": null,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"

src/libstore-tests/data/path-info/empty_impure.json

@@ -3,6 +3,7 @@
   "deriver": null,
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 0,
+  "provenance": null,
   "references": [],
   "registrationTime": null,
   "signatures": [],

src/libstore-tests/data/path-info/impure.json

@@ -3,6 +3,7 @@
   "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
   "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
   "narSize": 34878,
+  "provenance": null,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
     "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"

src/libstore/binary-cache-store.cc

@@ -302,7 +302,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(
     ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::shared_ptr<const Provenance> provenance)
 {
     std::optional<Hash> caHash;
     std::string nar;
@@ -364,6 +365,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(
             nar.first,
         };
         info.narSize = nar.second;
+        info.provenance = provenance;
         return info;
     })->path;
 }
@@ -448,7 +450,8 @@ StorePath BinaryCacheStore::addToStore(
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
     PathFilter & filter,
-    RepairFlag repair)
+    RepairFlag repair,
+    std::shared_ptr<const Provenance> provenance)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
@@ -474,6 +477,7 @@ StorePath BinaryCacheStore::addToStore(
             nar.first,
         };
         info.narSize = nar.second;
+        info.provenance = provenance;
         return info;
     })->path;
 }

src/libstore/binary-cache-store.hh

@@ -68,6 +68,9 @@ protected:
 
 public:
 
+    bool uriIsUsefulProvenance() override
+    { return true; }
+
     virtual bool fileExists(const std::string & path) = 0;
 
     virtual void upsertFile(const std::string & path,
@@ -129,7 +132,8 @@ public:
         ContentAddressMethod hashMethod,
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
-        RepairFlag repair) override;
+        RepairFlag repair,
+        std::shared_ptr<const Provenance> provenance) override;
 
     StorePath addToStore(
         std::string_view name,
@@ -138,7 +142,8 @@ public:
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         PathFilter & filter,
-        RepairFlag repair) override;
+        RepairFlag repair,
+        std::shared_ptr<const Provenance> provenance) override;
 
     void registerDrvOutput(const Realisation & info) override;
 

src/libstore/build/derivation-goal.cc

@@ -172,7 +172,7 @@ Goal::Co DerivationGoal::loadDerivation()
        things being garbage collected while we're busy. */
     worker.evalStore.addTempRoot(drvPath);
 
-    /* Get the derivation. It is probably in the eval store, but it might be inthe main store:
+    /* Get the derivation. It is probably in the eval store, but it might be in the main store:
 
          - Resolved derivation are resolved against main store realisations, and so must be stored there.
 
@@ -181,6 +181,7 @@ Goal::Co DerivationGoal::loadDerivation()
     for (auto * drvStore : { &worker.evalStore, &worker.store }) {
         if (drvStore->isValidPath(drvPath)) {
             drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
+            drvProvenance = drvStore->queryPathInfo(drvPath)->provenance;
             break;
         }
     }

src/libstore/build/derivation-goal.hh

@@ -67,6 +67,9 @@ struct DerivationGoal : public Goal
     /** The path of the derivation. */
     StorePath drvPath;
 
+    /** The provenance of the derivation, if any. */
+    std::shared_ptr<const Provenance> drvProvenance;
+
     /**
      * The goal for the corresponding resolved derivation
      */

src/libstore/daemon.cc

@@ -15,13 +15,16 @@
 #include "derivations.hh"
 #include "args.hh"
 #include "git.hh"
+#include "provenance.hh"
 
 #ifndef _WIN32 // TODO need graceful async exit support on Windows?
 # include "monitor-fd.hh"
 #endif
 
 #include <sstream>
 
+#include <nlohmann/json.hpp>
+
 namespace nix::daemon {
 
 Sink & operator << (Sink & sink, const Logger::Fields & fields)
@@ -398,6 +401,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             bool repairBool;
             conn.from >> repairBool;
             auto repair = RepairFlag{repairBool};
+            std::shared_ptr<const Provenance> provenance;
+            if (conn.features.contains(WorkerProto::featureProvenance))
+                provenance =
+                    nlohmann::json::parse(readString(conn.from))
+                    .template get<std::shared_ptr<const Provenance>>();
 
             logger->startWork();
             auto pathInfo = [&]() {
@@ -423,7 +431,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                     assert(false);
                 }
                 // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
-                auto path = store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair);
+                auto path = store->addToStoreFromDump(
+                    source,
+                    name,
+                    dumpMethod,
+                    contentAddressMethod,
+                    hashAlgo,
+                    refs,
+                    repair,
+                    provenance);
                 return store->queryPathInfo(path);
             }();
             logger->stopWork();