WIP: Git with refs types refactor

src/libstore/content-address.hh

@@ -4,6 +4,7 @@
 #include <variant>
 
 #include "hash.hh"
+#include "ipfs.hh"
 #include "path.hh"
 
 namespace nix {
@@ -30,9 +31,10 @@ struct FixedOutputHash {
     std::string printMethodAlgo() const;
 };
 
-struct IPFSHash {
-    Hash hash;
-};
+template<template <typename> class Ref>
+struct IPFSGitTreeNodeT;
+
+typedef IPFSGitTreeNodeT<IPFSHashWithOptValue> IPFSGitTreeNode;
 
 /*
   We've accumulated several types of content-addressed paths over the years;
@@ -48,7 +50,7 @@ struct IPFSHash {
 typedef std::variant<
     TextHash, // for paths computed by makeTextPath() / addTextToStore
     FixedOutputHash, // for path computed by makeFixedOutputPath
-    IPFSHash
+    IPFSHash<IPFSGitTreeNode>
 > LegacyContentAddress;
 
 /* Compute the prefix to the hash algorithm which indicates how the files were
@@ -139,44 +141,54 @@ struct FixedOutputInfo : FixedOutputHash {
     PathReferences<StorePath> references;
 };
 
-// pair of cid and name for ipfs references
-typedef std::pair<std::string, std::string> IPFSRef;
+template<typename Underlying>
+struct ContentAddressT;
 
-struct IPFSInfo {
-    Hash hash;
+template<template <typename> class Ref>
+struct IPFSGitTreeNodeT {
+	Hash gitTree;
     // References for the paths
-    PathReferences<IPFSRef> references;
+    PathReferences<ContentAddressT<Ref<IPFSGitTreeNodeT<Ref>>>> references;
 };
 
-// just the cid, which is a hash of ipfsinfo
-struct IPFSCid {
-    std::string cid;
-};
+// FIXME names
+typedef std::variant<
+    TextInfo,
+    FixedOutputInfo,
+    IPFSHashWithOptValue<IPFSGitTreeNode>
+> ContentAddressWithoutName;
 
-struct ContentAddress {
+template<typename Underlying>
+struct ContentAddressT {
     std::string name;
-    std::variant<
-        TextInfo,
-        FixedOutputInfo,
-        IPFSInfo,
-        IPFSCid
-    > info;
-
-    bool operator < (const ContentAddress & other) const
+    Underlying info;
+
+    bool operator < (const ContentAddressT<Underlying> & other) const
     {
         return name < other.name;
         // FIXME second field
     }
 };
 
+typedef ContentAddressT<ContentAddressWithoutName> ContentAddress;
+
 std::string renderContentAddress(ContentAddress ca);
 
 ContentAddress parseContentAddress(std::string_view rawCa);
 
-void to_json(nlohmann::json& j, const ContentAddress & c);
-void from_json(const nlohmann::json& j, ContentAddress & c);
+template<template <typename> class Ref>
+void to_json(nlohmann::json& j, const IPFSGitTreeNodeT<Ref> & c);
+template<template <typename> class Ref>
+void from_json(const nlohmann::json& j, IPFSGitTreeNodeT<Ref> & c);
+
+template<typename T>
+void to_json(nlohmann::json& j, const ContentAddressT<T> & c);
+template<typename T>
+void from_json(const nlohmann::json& j, ContentAddressT<T> & c);
 
-void to_json(nlohmann::json& j, const PathReferences<IPFSRef> & c);
-void from_json(const nlohmann::json& j, PathReferences<IPFSRef> & c);
+template<typename T>
+void to_json(nlohmann::json& j, const PathReferences<T> & c);
+template<typename T>
+void from_json(const nlohmann::json& j, PathReferences<T> & c);
 
 }

src/libstore/ipfs-binary-cache-store.cc

@@ -511,7 +511,7 @@ std::optional<std::string> IPFSBinaryCacheStore::getCidFromCA(ContentAddress ca)
             assert(ca_.hash.type == htSHA1);
             return "f01781114" + ca_.hash.to_string(Base16, false);
         }
-    } else if (std::holds_alternative<IPFSInfo>(ca.info))
+    } else if (std::holds_alternative<IPFSHashWithOptValue<IPFSGitTreeNode>>(ca.info))
         return computeIPFSCid(ca);
     else if (std::holds_alternative<IPFSCid>(ca.info))
         return std::get<IPFSCid>(ca.info).cid;
@@ -655,7 +655,7 @@ void IPFSBinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSo
 
             return;
         }
-    } else if (info.ca && std::holds_alternative<IPFSHash>(*info.ca)) {
+    } else if (info.ca && std::holds_alternative<IPFSHash<IPSFGitTreeNode>>(*info.ca)) {
         auto nar = make_ref<std::string>(narSource.drain());
 
         AutoDelete tmpDir(createTempDir(), true);

src/libstore/store-api.cc

@@ -215,74 +215,29 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons
         name);
 }
 
-static HashType getMultiHashTag(int tag)
+// FIXME move to content-address.cc
+Hash computeIPFSHash(const IPFSGitTreeNode & info)
 {
-    switch (tag) {
-    case 0x11: {
-        return htSHA1;
-    }
-    case 0x12: {
-        return htSHA256;
-    }
-    default: {
-        throw Error("tag '%i' is an unknown hash type", tag);
-    }
-    }
-}
-
-static std::vector<uint8_t> packMultihash(std::string cid)
-{
-    std::vector<uint8_t> result;
-    assert(cid[0] == 'f');
-    result.push_back(0x00);
-    result.push_back(std::stoi(cid.substr(1, 2), nullptr, 16));
-    result.push_back(std::stoi(cid.substr(3, 2), nullptr, 16));
-    result.push_back(std::stoi(cid.substr(5, 2), nullptr, 16));
-    result.push_back(std::stoi(cid.substr(7, 2), nullptr, 16));
-    HashType ht = getMultiHashTag(std::stoi(cid.substr(5, 2), nullptr, 16));
-    Hash hash = Hash::parseAny(cid.substr(9), ht);
-    for (unsigned int i = 0; i < hash.hashSize; i++)
-        result.push_back(hash.hash[i]);
-    return result;
-}
+    nlohmann::json j { info };
 
-Hash computeIPFSHash(const ContentAddress & info, HashType ht)
-{
-    nlohmann::json j = info;
+    auto pack = [&](auto & ref) {
+        ref = ref.template get<IPFSHash<IPFSGitTreeNode>>().packForCBOR();
+	};
 
     // replace {"/": ...} with packed multihash
     // ipfs converts automatically between the two
-    j.at("cid") = nlohmann::json::binary(packMultihash(j.at("cid").at("/").get<std::string>()), 42);
+    pack(j.at("gitTree"));
     for (auto & ref : j.at("references").at("references"))
-        ref.at("cid") = nlohmann::json::binary(packMultihash(ref.at("cid").at("/").get<std::string>()), 42);
+        pack(ref);
 
     std::vector<std::uint8_t> cbor = nlohmann::json::to_cbor(j);
-    return hashString(ht, std::string(cbor.begin(), cbor.end()));
+    return hashString(htSHA256, std::string(cbor.begin(), cbor.end()));
 }
 
-std::string computeIPFSCid(const ContentAddress & info)
+StorePath Store::makeIPFSPath(std::string_view name, IPFSHash<IPFSGitTreeNode> hash) const
 {
-    return "f01711220" + computeIPFSHash(info, htSHA256).to_string(Base16, false);
-}
-
-StorePath Store::makeIPFSPath(IPFSRef ref) const
-{
-    string type = "ipfs";
-
-    // copy pase from makeStorePath
-    string s = type + ":" + ref.first + ":" + storeDir + ":" + std::string(ref.second);
-    auto h = compressHash(hashString(htSHA256, s), 20);
-    return StorePath(h, ref.second);
-}
-
-StorePath Store::makeIPFSPath(std::string cid, std::string name) const
-{
-    return makeIPFSPath(IPFSRef(cid, name));
-}
-
-StorePath Store::makeIPFSPath(const ContentAddress & info) const
-{
-    return makeIPFSPath(computeIPFSCid(info), info.name);
+	// `to_string` use IPFS-style multi* encoding for forward-compat.
+    return makeStorePath("ipfs-git", hash.hash, name);
 }
 
 StorePath Store::makeFixedOutputPathFromCA(const ContentAddress & info) const
@@ -295,12 +250,9 @@ StorePath Store::makeFixedOutputPathFromCA(const ContentAddress & info) const
         [&](FixedOutputInfo foi) {
             return makeFixedOutputPath(info.name, foi);
         },
-        [&](IPFSInfo io) {
-            return makeIPFSPath(info);
+        [&](IPFSHash<IPFSGitTreeNode> ih) {
+            return makeIPFSPath(info.name, ih);
         },
-        [&](IPFSCid ic) {
-            return makeIPFSPath(ic.cid, info.name);
-        }
     }, info.info);
 }
 
@@ -974,15 +926,17 @@ std::optional<ContentAddress> ValidPathInfo::fullContentAddressOpt() const
                 TextInfo info { th };
                 assert(!hasSelfReference);
                 info.references = references;
-                return std::variant<TextInfo, FixedOutputInfo, IPFSInfo, IPFSCid> { info };
+                return ContentAddressWithoutName { info };
             },
             [&](FixedOutputHash foh) {
                 FixedOutputInfo info { foh };
                 info.references = static_cast<PathReferences<StorePath>>(*this);
-                return std::variant<TextInfo, FixedOutputInfo, IPFSInfo, IPFSCid> { info };
+                return ContentAddressWithoutName { info };
             },
-            [&](IPFSHash io) {
-                return std::variant<TextInfo, FixedOutputInfo, IPFSInfo, IPFSCid> { IPFSCid { "f01711220" + io.hash.to_string(Base16, false) } };
+            [&](IPFSHash<IPFSGitTreeNode> io) {
+                return ContentAddressWithoutName {
+                    IPFSHashWithOptValue<IPFSGitTreeNode> { io },
+                };
             },
         }, *ca),
     };
@@ -1047,14 +1001,13 @@ ValidPathInfo::ValidPathInfo(
             *(static_cast<PathReferences<StorePath> *>(this)) = foi.references;
             this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) };
         },
-        [this, &store, info](IPFSInfo foi) {
-            this->hasSelfReference = foi.references.hasSelfReference;
-            for (auto & ref : foi.references.references)
-                this->references.insert(store.makeIPFSPath(ref));
-            this->ca = IPFSHash { computeIPFSHash(info, htSHA256) };
-        },
-        [](IPFSCid foi) {
-            throw Error("cannot make a valid path from a cid");
+        [this, &store](IPFSHashWithOptValue<IPFSGitTreeNode> ih) {
+            this->ca = static_cast<IPFSHash<IPFSGitTreeNode>>(ih);
+            if (ih.value) {
+                this->hasSelfReference = ih.value->references.hasSelfReference;
+                for (auto & ref : ih.value->references.references)
+                    this->references.insert(store.makeIPFSPath(ref.name, ref.info));
+            }
         },
     }, std::move(info.info));
 }

src/libstore/store-api.hh

@@ -266,7 +266,7 @@ public:
 
     StorePath makeIPFSPath(const ContentAddress & info) const;
 
-    StorePath makeIPFSPath(IPFSRef ref) const;
+    StorePath makeIPFSPath(std::string_view name, IPFSHash<IPFSGitTreeNode> ref) const;
 
     StorePath makeIPFSPath(std::string cid, std::string name) const;
 
@@ -771,8 +771,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri)
 
 std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
 
-Hash computeIPFSHash(const ContentAddress & info);
-std::string computeIPFSCid(const ContentAddress & info);
+Hash computeIPFSHash(const IPFSGitTreeNode & info);
 
 
 }

src/libutil/hash.hh

@@ -31,6 +31,10 @@ struct Hash
 
     HashType type;
 
+	/* Using and arbitrary dummy value is bad, but very useful for
+	   <nlohmann/json> so we do it anways */
+	Hash() : Hash { htSHA256 } { }
+
     /* Create a zero-filled hash object. */
     Hash(HashType type);
 

src/libutil/ipfs.cc

@@ -0,0 +1,41 @@
+#include "ipfs.hh"
+
+namespace nix::untyped {
+
+// "f01711220" base-16, sha-256
+// "f01781114" base-16, sha-1
+
+std::string toString(Hash hash) {
+    std::string prefix;
+    switch (hash.type) {
+    case htSHA1: prefix = "f01711220";
+    case htSHA256: prefix = "f01781114";
+    default: throw Error("hash type '%s' we don't yet export to IPFS", printHashType(hash.type));
+    }
+    return prefix + hash.to_string(Base16, false);
+}
+
+Hash fromString(std::string_view cid) {
+    auto prefix = cid.substr(0, 9);
+    HashType algo = prefix == "f01711220" ? htSHA256
+        : prefix == "f01781114" ? htSHA1
+        : throw Error("cid '%s' is wrong type for ipfs hash", cid);
+    return Hash::parseNonSRIUnprefixed(cid.substr(9), algo);
+}
+
+std::vector<uint8_t> pack(Hash hash)
+{
+	auto cid = toString(hash);
+    std::vector<uint8_t> result;
+    assert(cid[0] == 'f');
+    result.push_back(0x00);
+    result.push_back(std::stoi(cid.substr(1, 2), nullptr, 16));
+    result.push_back(std::stoi(cid.substr(3, 2), nullptr, 16));
+    result.push_back(std::stoi(cid.substr(5, 2), nullptr, 16));
+    result.push_back(std::stoi(cid.substr(7, 2), nullptr, 16));
+    for (unsigned int i = 0; i < hash.hashSize; i++)
+        result.push_back(hash.hash[i]);
+    return result;
+}
+
+}