valid path info ca proper datatype

src/libexpr/get-drvs.cc

@@ -1,7 +1,7 @@
 #include "get-drvs.hh"
 #include "util.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 #include <cstring>
 #include <regex>

src/libexpr/primops/context.cc

@@ -1,6 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 namespace nix {
 

src/libfetchers/tarball.cc

@@ -70,7 +70,10 @@ DownloadFileResult downloadFile(
         ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
         info.narHash = hashString(HashType::SHA256, *sink.s);
         info.narSize = sink.s->size();
-        info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
+        info.ca = FileSystemHash {
+            FileIngestionMethod::Flat,
+            hash,
+        };
         auto source = StringSource { *sink.s };
         store->addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);

src/libstore/build.cc

@@ -3708,7 +3708,7 @@ void DerivationGoal::registerOutputs()
         /* Check that fixed-output derivations produced the right
            outputs (i.e., the content hash should match the specified
            hash). */
-        std::string ca;
+        std::optional<ContentAddress> ca;
 
         if (fixedOutput) {
 
@@ -3757,7 +3757,7 @@ void DerivationGoal::registerOutputs()
             else
                 assert(worker.store.parseStorePath(path) == dest);
 
-            ca = makeFixedOutputCA(i.second.hash->method, h2);
+            ca = FileSystemHash { i.second.hash->method, h2 };
         }
 
         /* Get rid of all weird permissions.  This also checks that
@@ -3830,7 +3830,10 @@ void DerivationGoal::registerOutputs()
         info.ca = ca;
         worker.store.signPathInfo(info);
 
-        if (!info.references.empty()) info.ca.clear();
+        if (!info.references.empty()) {
+            // FIXME don't we have an experimental feature for fixed output with references?
+            info.ca = {};
+        }
 
         infos.emplace(i.first, std::move(info));
     }

src/libstore/content-address.cc

@@ -0,0 +1,83 @@
+#include "content-address.hh"
+
+namespace nix {
+
+std::string FileSystemHash::printMethodAlgo() const {
+    return makeFileIngestionPrefix(method) + printHashType(*hash.type);
+}
+
+std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
+    switch (m) {
+    case FileIngestionMethod::Flat:
+        return "";
+    case FileIngestionMethod::Recursive:
+        return "r:";
+    case FileIngestionMethod::Git:
+        return "git:";
+    }
+    abort();
+}
+
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+{
+    return "fixed:"
+        + makeFileIngestionPrefix(method)
+        + hash.to_string();
+}
+
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+std::string renderContentAddress(ContentAddress ca) {
+    return std::visit(overloaded {
+        [](TextHash th) {
+            return "text:" + th.hash.to_string();
+        },
+        [](FileSystemHash fsh) {
+            return makeFixedOutputCA(fsh.method, fsh.hash);
+        }
+    }, ca);
+}
+
+ContentAddress parseContentAddress(std::string_view rawCa) {
+    auto prefixSeparator = rawCa.find(':');
+    if (prefixSeparator != string::npos) {
+        auto prefix = string(rawCa, 0, prefixSeparator);
+        if (prefix == "text") {
+            auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+            Hash hash = Hash(string(hashTypeAndHash));
+            if (*hash.type != HashType::SHA256) {
+                throw Error("parseContentAddress: the text hash should have type SHA256");
+            }
+            return TextHash { hash };
+        } else if (prefix == "fixed") {
+            // This has to be an inverse of makeFixedOutputCA
+            auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+            if (methodAndHash.substr(0, 2) == "r:") {
+                std::string_view hashRaw = methodAndHash.substr(2, string::npos);
+                return FileSystemHash { FileIngestionMethod::Recursive, Hash(string(hashRaw)) };
+            } else if (methodAndHash.substr(0, 4) == "git:") {
+                std::string_view hashRaw = methodAndHash.substr(4, string::npos);
+                return FileSystemHash { FileIngestionMethod::Git, Hash(string(hashRaw)) };
+            } else {
+                std::string_view hashRaw = methodAndHash;
+                return FileSystemHash { FileIngestionMethod::Flat, Hash(string(hashRaw)) };
+            }
+        } else {
+            throw Error("parseContentAddress: format not recognized; has to be text or fixed");
+        }
+    } else {
+        throw Error("Not a content address because it lacks an appropriate prefix");
+    }
+};
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
+    return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
+};
+
+std::string renderContentAddress(std::optional<ContentAddress> ca) {
+    return ca ? renderContentAddress(*ca) : "";
+}
+
+}

src/libstore/content-address.hh

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <variant>
+#include "hash.hh"
+
+namespace nix {
+
+enum struct FileIngestionMethod : uint8_t {
+    Flat,
+    Recursive,
+    Git,
+};
+
+
+struct TextHash {
+    Hash hash;
+    TextHash(const TextHash &) = default;
+    TextHash(TextHash &&) = default;
+    TextHash & operator = (const TextHash &) = default;
+};
+
+/// Pair of a hash, and how the file system was ingested
+struct FileSystemHash {
+    FileIngestionMethod method;
+    Hash hash;
+    FileSystemHash(FileIngestionMethod method, Hash hash)
+        : method(std::move(method))
+        , hash(std::move(hash))
+    { }
+    FileSystemHash(const FileSystemHash &) = default;
+    FileSystemHash(FileSystemHash &&) = default;
+    FileSystemHash & operator = (const FileSystemHash &) = default;
+    std::string printMethodAlgo() const;
+};
+
+/*
+  We've accumulated several types of content-addressed paths over the years;
+  fixed-output derivations support multiple hash algorithms and serialisation
+  methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms:
+
+  * ‘text:sha256:<sha256 hash of file contents>’: For paths
+    computed by makeTextPath() / addTextToStore().
+
+  * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
+    makeFixedOutputPath() / addToStore().
+*/
+typedef std::variant<
+    TextHash, // for paths computed by makeTextPath() / addTextToStore
+    FileSystemHash // for path computed by makeFixedOutputPath
+> ContentAddress;
+
+/* Compute the prefix to the hash algorithm which indicates how the files were
+   ingested. */
+std::string makeFileIngestionPrefix(const FileIngestionMethod m);
+
+/* Compute the content-addressability assertion (ValidPathInfo::ca)
+   for paths created by makeFixedOutputPath() / addToStore(). */
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
+
+std::string renderContentAddress(ContentAddress ca);
+
+std::string renderContentAddress(std::optional<ContentAddress> ca);
+
+ContentAddress parseContentAddress(std::string_view rawCa);
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
+
+}

src/libstore/daemon.cc

@@ -663,7 +663,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
                 to << info->ultimate
                    << info->sigs
-                   << info->ca;
+                   << renderContentAddress(info->ca);
             }
         } else {
             assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
@@ -721,7 +721,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         info.references = readStorePaths<StorePathSet>(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
-        from >> info.ca >> repair >> dontCheckSigs;
+        info.ca = parseContentAddressOpt(readString(from));
+        from >> repair >> dontCheckSigs;
         if (!trusted && dontCheckSigs)
             dontCheckSigs = false;
         if (!trusted)

src/libstore/derivations.cc

@@ -8,11 +8,6 @@
 
 namespace nix {
 
-std::string FileSystemHash::printMethodAlgo() const {
-    return makeFileIngestionPrefix(method) + printHashType(*hash.type);
-}
-
-
 BasicDerivation::BasicDerivation(const BasicDerivation & other)
     : platform(other.platform)
     , builder(other.builder)

src/libstore/derivations.hh

@@ -1,8 +1,9 @@
 #pragma once
 
+#include "path.hh"
 #include "types.hh"
 #include "hash.hh"
-#include "store-api.hh"
+#include "content-address.hh"
 
 #include <map>
 
@@ -12,20 +13,6 @@ namespace nix {
 
 /* Abstract syntax of derivations. */
 
-/// Pair of a hash, and how the file system was ingested
-struct FileSystemHash {
-    FileIngestionMethod method;
-    Hash hash;
-    FileSystemHash(FileIngestionMethod method, Hash hash)
-        : method(std::move(method))
-        , hash(std::move(hash))
-    { }
-    FileSystemHash(const FileSystemHash &) = default;
-    FileSystemHash(FileSystemHash &&) = default;
-    FileSystemHash & operator = (const FileSystemHash &) = default;
-    std::string printMethodAlgo() const;
-};
-
 struct DerivationOutput
 {
     StorePath path;
@@ -90,6 +77,7 @@ struct Derivation : BasicDerivation
 
 class Store;
 
+enum RepairFlag : bool { NoRepair = false, Repair = true };
 
 /* Write a derivation to the Nix store, and return its path. */
 StorePath writeDerivation(ref<Store> store,

src/libstore/legacy-ssh-store.cc

@@ -114,7 +114,7 @@ struct LegacySSHStore : public Store
             if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
                 auto s = readString(conn->from);
                 info->narHash = s.empty() ? Hash() : Hash(s);
-                conn->from >> info->ca;
+                info->ca = parseContentAddressOpt(readString(conn->from));
                 info->sigs = readStrings<StringSet>(conn->from);
             }
 
@@ -146,7 +146,7 @@ struct LegacySSHStore : public Store
                 << info.narSize
                 << info.ultimate
                 << info.sigs
-                << info.ca;
+                << renderContentAddress(info.ca);
             try {
                 copyNAR(source, conn->to);
             } catch (...) {

src/libstore/local-store.cc

@@ -577,7 +577,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
-    if (info.ca != "" && !info.isContentAddressed(*this))
+    if (info.ca.has_value() && !info.isContentAddressed(*this))
         throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
             printStorePath(info.path));
 
@@ -589,7 +589,7 @@ uint64_t LocalStore::addValidPath(State & state,
         (info.narSize, info.narSize != 0)
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
-        (info.ca, !info.ca.empty())
+        (renderContentAddress(info.ca), (bool) info.ca)
         .exec();
     uint64_t id = sqlite3_last_insert_rowid(state.db);
 
@@ -663,7 +663,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
             if (s) info->sigs = tokenizeString<StringSet>(s, " ");
 
             s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
-            if (s) info->ca = s;
+            if (s) info->ca = parseContentAddressOpt(s);
 
             /* Get the references. */
             auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
@@ -686,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
         (info.narHash.to_string(Base::Base16))
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
-        (info.ca, !info.ca.empty())
+        (renderContentAddress(info.ca), (bool) info.ca)
         (printStorePath(info.path))
         .exec();
 }
@@ -1000,15 +1000,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             deletePath(realPath);
 
-            if (info.ca != "" &&
-                !((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
-                    || info.references.empty()))
+            // text hashing has long been allowed to have non-self-references because it is used for drv files.
+            bool refersToSelf = info.references.count(info.path) > 0;
+            if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
                 settings.requireExperimentalFeature("ca-references");
 
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
             std::unique_ptr<AbstractHashSink> hashSink;
-            if (info.ca == "" || !info.references.count(info.path))
+            if (!info.ca.has_value() || !info.references.count(info.path))
                 hashSink = std::make_unique<HashSink>(HashType::SHA256);
             else
                 hashSink = std::make_unique<HashModuloSink>(HashType::SHA256, storePathToHash(printStorePath(info.path)));
@@ -1019,7 +1019,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
                 return n;
             });
 
-            if (hasPrefix(info.ca, "fixed:git:"))
+            auto p = info.ca ? std::get_if<FileSystemHash>(&*info.ca) : NULL;
+            if (p && p->method == FileIngestionMethod::Git)
                 restoreGit(realPath, wrapperSource, realStoreDir, storeDir);
             else
                 restorePath(realPath, wrapperSource);
@@ -1110,7 +1111,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
             ValidPathInfo info(dstPath.clone());
             info.narHash = hash.first;
             info.narSize = hash.second;
-            info.ca = makeFixedOutputCA(method, h);
+            info.ca = FileSystemHash { method, h };
             registerValidPath(info);
         }
 
@@ -1195,7 +1196,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
             info.narHash = narHash;
             info.narSize = sink.s->size();
             info.references = cloneStorePathSet(references);
-            info.ca = "text:" + hash.to_string();
+            info.ca = TextHash { .hash = hash };
             registerValidPath(info);
         }
 
@@ -1303,7 +1304,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 printMsg(Verbosity::Talkative, "checking contents of '%s'", printStorePath(i));
 
                 std::unique_ptr<AbstractHashSink> hashSink;
-                if (info->ca == "" || !info->references.count(info->path))
+                if (!info->ca || !info->references.count(info->path))
                     hashSink = std::make_unique<HashSink>(*info->narHash.type);
                 else
                     hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, storePathToHash(printStorePath(info->path)));