dir_shard

cmake/patch.py

@@ -151,7 +151,7 @@ def electron_version(self, branch='main'):
     def unavailable(self):
         avail = list(map(as_int, self.available()))
         version_set = {}
-        fuzz = 112
+        fuzz = 113
         def check(version, version_set, s):
             i = as_int(version)
             by = (fuzz,0)

cmake/valgrind.suppressions.txt

@@ -7,7 +7,7 @@
 }
 {
    DontAskMeWhyGetEnvDoesThis
-   Memcheck:Addr8
+   Memcheck:Cond
    ...
    fun:getenv
    ...

library/include/ipfs_client/ipld/dag_node.h

@@ -19,12 +19,14 @@
 
 namespace ipfs {
 class Block;
+class ValidatedIpns;
 }
 
 namespace ipfs::ipld {
 
 class DagNode;
 using NodePtr = std::shared_ptr<DagNode>;
+class DirShard;
 
 struct MoreDataNeeded {
   std::vector<std::string> ipfs_abs_paths_;
@@ -47,8 +49,10 @@ class DagNode : public std::enable_shared_from_this<DagNode> {
 
   virtual NodePtr rooted();
   virtual NodePtr deroot();
+  virtual DirShard* as_hamt();  // Wish I had access to dynamic_cast
 
   static NodePtr fromBlock(Block const&);
+  static NodePtr fromIpnsRecord(ValidatedIpns const&);
 };
 }  // namespace ipfs::ipld
 

library/src/ipfs_client/gw/gateway_request.cc

@@ -12,7 +12,9 @@ std::shared_ptr<Self> Self::fromIpfsPath(ipfs::SlashDelimited p) {
     result->path = p.pop_all();
     result->type = result->path.empty() ? Type::Block : Type::Car;
   } else if (name_space == "ipns") {
-    LOG(ERROR) << "TODO ipns";
+    result->main_param = p.pop();
+    result->path = p.pop_all();
+    result->type = Type::Ipns;
   } else {
     LOG(FATAL) << "Unsupported namespace in ipfs path: /" << name_space << '/'
                << p.pop_all();

library/src/ipfs_client/ipld/dag_node.cc

@@ -1,11 +1,14 @@
 #include <ipfs_client/ipld/dag_node.h>
 
 #include "chunk.h"
+#include "directory_shard.h"
+#include "ipns_name.h"
 #include "root.h"
 #include "small_directory.h"
 #include "unixfs_file.h"
 
 #include <ipfs_client/dag_block.h>
+#include <ipfs_client/ipns_record.h>
 
 #include "log_macros.h"
 
@@ -23,6 +26,13 @@ std::shared_ptr<Node> Node::fromBlock(ipfs::Block const& block) {
     case Block::Type::File:
       result = std::make_shared<UnixfsFile>();
       break;
+    case Block::Type::HAMTShard:
+      if (block.fsdata().has_fanout()) {
+        result = std::make_shared<DirShard>(block.fsdata().fanout());
+      } else {
+        result = std::make_shared<DirShard>();
+      }
+      break;
     case Block::Type::Invalid:
       LOG(ERROR) << "Invalid block.";
       return result;
@@ -37,9 +47,16 @@ std::shared_ptr<Node> Node::fromBlock(ipfs::Block const& block) {
   return result;
 }
 
+auto Node::fromIpnsRecord(ipfs::ValidatedIpns const& v) -> NodePtr {
+  return std::make_shared<IpnsName>(v.value);
+}
+
 std::shared_ptr<Node> Node::deroot() {
   return shared_from_this();
 }
 std::shared_ptr<Node> Node::rooted() {
   return std::make_shared<Root>(shared_from_this());
 }
+auto Node::as_hamt() -> DirShard* {
+  return nullptr;
+}

library/src/ipfs_client/ipld/directory_shard.cc

@@ -0,0 +1,100 @@
+#include "directory_shard.h"
+
+#include "log_macros.h"
+
+#include <vocab/endian.h>
+
+#include <smhasher/MurmurHash3.h>
+
+#define ABSL_USES_STD_STRING_VIEW
+#include <absl/strings/match.h>
+
+#include <iomanip>
+#include <sstream>
+
+using namespace std::literals;
+
+using Self = ipfs::ipld::DirShard;
+
+auto Self::resolve(ipfs::SlashDelimited relpath,
+                   ipfs::ipld::DagNode::BlockLookup blu,
+                   std::string& to_here) -> ResolveResult {
+  if (!relpath) {
+    // TODO check if index.html is present and if not implement indexing
+    auto result = resolve("index.html"sv, blu, to_here);
+    auto resp = std::get_if<Response>(&result);
+    if (resp) {
+      resp->mime_ = "text/html";
+    }
+    return result;
+  }
+  auto name = relpath.pop();
+  auto hash = hexhash(name);
+  return resolve_internal(hash.begin(), hash.end(), name, relpath, blu,
+                          to_here);
+}
+auto Self::resolve_internal(ipfs::ipld::DirShard::HashIter hash_b,
+                            ipfs::ipld::DirShard::HashIter hash_e,
+                            std::string_view element_name,
+                            ipfs::SlashDelimited path_after_dir,
+                            ipfs::ipld::DagNode::BlockLookup blu,
+                            std::string& path_to_dir) -> ResolveResult {
+  for (auto& [name, link] : links_) {
+    if (hash_b != hash_e && !absl::StartsWith(name, *hash_b)) {
+      continue;
+    }
+    if (!link.node) {
+      link.node = blu(link.cid);
+    }
+    if (!link.node) {
+      return MoreDataNeeded{std::vector{"/ipfs/" + link.cid}};
+    }
+    if (absl::EndsWith(name, element_name)) {
+      return link.node->resolve(path_after_dir, blu, path_to_dir);
+    }
+    auto downcast = link.node->as_hamt();
+    if (downcast) {
+      if (hash_b == hash_e) {
+        LOG(ERROR) << "Ran out of hash bits.";
+        return ProvenAbsent{};
+      }
+      return downcast->resolve_internal(std::next(hash_b), hash_e, element_name,
+                                        path_after_dir, blu, path_to_dir);
+    } else {
+      return ProvenAbsent{};
+    }
+  }
+  return ProvenAbsent{};
+}
+std::vector<std::string> Self::hexhash(std::string_view path_element) const {
+  auto hex_width = 0U;
+  for (auto x = fanout_; (x >>= 4); ++hex_width)
+    ;
+  std::array<std::uint64_t, 2> digest = {0U, 0U};
+  MurmurHash3_x64_128(path_element.data(), path_element.size(), 0,
+                      digest.data());
+  auto corrected_digest = htobe64(digest[0]);
+  VLOG(1) << "Hash: " << digest[0] << ' ' << digest[1] << " -> "
+          << corrected_digest;
+  std::vector<std::string> result;
+  for (auto d : digest) {
+    auto hash_bits = htobe64(d);
+    while (hash_bits) {
+      // 2. Pop the log2(fanout_) lowest bits from the path component hash
+      // digest,...
+      auto popped = hash_bits % fanout_;
+      hash_bits /= fanout_;
+      std::ostringstream oss;
+      // ... then hex encode (using 0-F) using little endian those bits ...
+      oss << std::setfill('0') << std::setw(hex_width) << std::uppercase
+          << std::hex << popped;
+      result.push_back(oss.str());
+    }
+  }
+  return result;
+}
+
+Self::DirShard(std::uint64_t fanout) : fanout_{fanout} {}
+Self* Self::as_hamt() {
+  return this;
+}

library/src/ipfs_client/ipld/directory_shard.h

@@ -0,0 +1,27 @@
+#ifndef IPFS_DIRECTORY_SHARD_H_
+#define IPFS_DIRECTORY_SHARD_H_ 1
+
+#include <ipfs_client/ipld/dag_node.h>
+
+namespace ipfs::ipld {
+class DirShard : public DagNode {
+  std::uint64_t const fanout_;
+
+  ResolveResult resolve(SlashDelimited, BlockLookup, std::string&) override;
+  DirShard* as_hamt() override;
+
+  std::vector<std::string> hexhash(std::string_view path_element) const;
+  using HashIter = std::vector<std::string>::const_iterator;
+  ResolveResult resolve_internal(HashIter,
+                                 HashIter,
+                                 std::string_view,
+                                 SlashDelimited,
+                                 BlockLookup,
+                                 std::string&);
+
+ public:
+  explicit DirShard(std::uint64_t fanout = 256UL);
+};
+}  // namespace ipfs::ipld
+
+#endif  // IPFS_DIRECTORY_SHARD_H_

library/src/ipfs_client/ipld/ipns_name.cc

@@ -0,0 +1,22 @@
+#include "ipns_name.h"
+
+#include "log_macros.h"
+
+using Self = ipfs::ipld::IpnsName;
+
+Self::IpnsName(std::string target_abs_path) : target_path_{target_abs_path} {}
+
+auto Self::resolve(ipfs::SlashDelimited path,
+                   ipfs::ipld::DagNode::BlockLookup blu,
+                   std::string& up_to_here) -> ResolveResult {
+  if (!target_) {
+    SlashDelimited t{target_path_};
+    t.pop();  // Discard namespace, though realistically it's going to be ipfs
+              // basically all the time
+    target_ = blu(std::string{t.pop()});
+  }
+  if (target_) {
+    return target_->resolve(path, blu, up_to_here);
+  }
+  return MoreDataNeeded{std::vector{target_path_}};
+}

library/src/ipfs_client/ipld/ipns_name.h

@@ -0,0 +1,20 @@
+#ifndef IPFS_IPLD_IPNS_NAME_H_
+#define IPFS_IPLD_IPNS_NAME_H_
+
+#include "ipfs_client/ipld/dag_node.h"
+
+namespace ipfs::ipld {
+class IpnsName : public DagNode {
+  std::string const target_path_;
+  NodePtr target_;
+
+  ResolveResult resolve(SlashDelimited path,
+                        BlockLookup,
+                        std::string& up_to_here) override;
+
+ public:
+  IpnsName(std::string target_abs_path);
+};
+}  // namespace ipfs::ipld
+
+#endif  // IPFS_IPLD_IPNS_NAME_H_

library/src/ipfs_client/ipld/small_directory.cc

@@ -3,15 +3,50 @@
 #include "ipfs_client/generated_directory_listing.h"
 #include "ipfs_client/path2url.h"
 
+using namespace std::literals;
+
 using Self = ipfs::ipld::SmallDirectory;
 
-auto Self::resolve(ipfs::SlashDelimited path,
-                   ipfs::ipld::DagNode::BlockLookup blu,
-                   std::string& to_here) -> ResolveResult {
+namespace {
+ipfs::ipld::NodePtr& node(ipfs::ipld::Link& l,
+                          ipfs::ipld::DagNode::BlockLookup f) {
+  if (!l.node) {
+    l.node = f(l.cid);
+  }
+  return l.node;
+}
+ipfs::ipld::NodePtr& node(std::pair<std::string, ipfs::ipld::Link>& l,
+                          ipfs::ipld::DagNode::BlockLookup f) {
+  return node(l.second, f);
+}
+std::string const& name(std::pair<std::string, ipfs::ipld::Link> const& p) {
+  return p.first;
+}
+std::string const& cid(std::pair<std::string, ipfs::ipld::Link> const& p) {
+  return p.second.cid;
+}
+}  // namespace
+
+auto Self::resolve(SlashDelimited path, BlockLookup blu, std::string& to_here)
+    -> ResolveResult {
   if (!path) {
     GeneratedDirectoryListing index_html{path2url(to_here)};
     for (auto& l : links_) {
-      index_html.AddEntry(l.first);
+      if (name(l) != "index.html") {
+        index_html.AddEntry(name(l));
+      } else {
+        auto& n = node(l, blu);
+        if (n) {
+          auto result = n->resolve(""sv, blu, to_here.append("/index.html"));
+          auto resp = std::get_if<Response>(&result);
+          if (resp) {
+            resp->mime_ = "text/html";
+          }
+          return result;
+        } else {
+          return MoreDataNeeded{std::vector{"/ipfs/" + cid(l)}};
+        }
+      }
     }
     return Response{"text/html", 200, index_html.Finish(), ""};
   }
@@ -22,19 +57,15 @@ auto Self::resolve(ipfs::SlashDelimited path,
   if (links_.end() == it) {
     return ProvenAbsent{};
   }
-  auto& entry_name = it->first;
-  auto& link = it->second;
-  if (!link.node) {
-    link.node = blu(link.cid);
-  }
-  if (link.node) {
+  auto& nod = node(*it, blu);
+  if (nod) {
     if (to_here.back() != '/') {
       to_here.push_back('/');
     }
-    to_here.append(entry_name);
-    return link.node->resolve(path, blu, to_here);
+    to_here.append(name);
+    return nod->resolve(path, blu, to_here);
   } else {
-    return MoreDataNeeded{std::vector{"/ipfs/" + link.cid}};
+    return MoreDataNeeded{std::vector{"/ipfs/" + cid(*it)}};
   }
 }