feat: ipfs-unixfs get or "walk over anything"

.github/workflows/conformance.yml

@@ -80,8 +80,8 @@ jobs:
         if: matrix.platform.cross == false
         run: cargo build --workspace
 
-      - name: npm install
-        run: npm install
+      - name: Setup conformance tests
+        run: ./setup.sh
         working-directory: ./conformance
 
       - name: Symlink executable

http/Cargo.toml

@@ -31,6 +31,9 @@ warp = "0.2.2"
 async-stream = "0.2.1"
 pin-project = "0.4.8"
 url = "2.1.1"
+tar = { version = "0.4.28", default-features = false }
+bytes = "0.5.4"
 
 [dev-dependencies]
 hex = "0.4.2"
+hex-literal = "0.2.1"

http/src/v0.rs

@@ -70,8 +70,8 @@ pub fn routes<T: IpfsTypes>(
         dag::put(ipfs),
         dag::resolve(ipfs),
         warp::path!("dht" / ..).and_then(not_implemented),
-        warp::path!("get").and_then(not_implemented),
         root_files::cat(ipfs),
+        root_files::get(ipfs),
         warp::path!("key" / ..).and_then(not_implemented),
         warp::path!("name" / ..).and_then(not_implemented),
         warp::path!("object" / ..).and_then(not_implemented),

http/src/v0/refs.rs

@@ -51,12 +51,18 @@ async fn refs_inner<T: IpfsTypes>(
         formatter
     );
 
-    let paths = opts
+    let mut paths = opts
         .arg
         .iter()
         .map(|s| IpfsPath::try_from(s.as_str()).map_err(StringError::from))
         .collect::<Result<Vec<_>, _>>()?;
 
+    for path in paths.iter_mut() {
+        // this is needed because the paths should not error on matching on the final Data segment,
+        // it just becomes projected as `Loaded::Raw(_)`, however such items can have no links.
+        path.set_follow_dagpb_data(true);
+    }
+
     let st = refs_paths(ipfs, paths, max_depth, opts.unique)
         .await
         .map_err(|e| {
@@ -241,7 +247,7 @@ pub async fn walk_path<T: IpfsTypes>(
     ipfs: &Ipfs<T>,
     mut path: IpfsPath,
 ) -> Result<(Cid, Loaded, Vec<String>), WalkError> {
-    use ipfs::unixfs::ll::MaybeResolved;
+    use ipfs::unixfs::ll::{MaybeResolved, ResolveError};
 
     let mut current = path.take_root().unwrap();
 
@@ -280,6 +286,13 @@ pub async fn walk_path<T: IpfsTypes>(
                 Ok(MaybeResolved::NotFound) => {
                     return handle_dagpb_not_found(current, &data, needle, &path)
                 }
+                Err(ResolveError::UnexpectedType(_)) => {
+                    // the conformance tests use a path which would end up going through a file
+                    // and the returned error string is tested against listed alternatives.
+                    // unexpected type is not one of them.
+                    let e = WalkFailed::from(path::WalkFailed::UnmatchedNamedLink(needle));
+                    return Err(WalkError::from((e, current)));
+                }
                 Err(e) => return Err(WalkError::from((WalkFailed::from(e), current))),
             };
 
@@ -360,10 +373,12 @@ fn handle_dagpb_not_found(
     needle: String,
     path: &IpfsPath,
 ) -> Result<(Cid, Loaded, Vec<String>), WalkError> {
-    if needle == "Data" && path.len() == 0 {
-        // /dag/resolve needs to "resolve through" a dag-pb node down to the "just
-        // data" even though we do not need to extract it ... however this might be
-        // good to just filter with refs, as no refs of such path exist
+    use ipfs::unixfs::ll::dagpb::node_data;
+
+    if needle == "Data" && path.len() == 0 && path.follow_dagpb_data() {
+        // /dag/resolve needs to "resolve through" a dag-pb node down to the "just data" even
+        // though we do not need to extract it ... however this might be good to just filter with
+        // refs, as no refs of such path can exist as the links are in the outer structure.
         //
         // testing with go-ipfs 0.5 reveals that dag resolve only follows links
         // which are actually present in the dag-pb, not numeric links like Links/5
@@ -372,7 +387,7 @@ fn handle_dagpb_not_found(
         // comment on this special casing: there cannot be any other such
         // special case as the Links do not work like Data so while this is not
         // pretty, it's not terrible.
-        let data = ipfs::unixfs::ll::dagpb::node_data(&data)
+        let data = node_data(&data)
             .expect("already parsed once, second time cannot fail")
             .unwrap_or_default();
         Ok((at, Loaded::Ipld(Ipld::Bytes(data.to_vec())), vec![needle]))

http/src/v0/refs/path.rs

@@ -34,6 +34,9 @@ pub struct IpfsPath {
     /// Option to support moving the cid
     root: Option<Cid>,
     path: std::vec::IntoIter<String>,
+    /// True by default, to allow "finding" `Data` under dag-pb node
+    /// TODO: document why this matters
+    follow_dagpb_data: bool,
 }
 
 impl From<Cid> for IpfsPath {
@@ -43,6 +46,7 @@ impl From<Cid> for IpfsPath {
         IpfsPath {
             root: Some(root),
             path: Vec::new().into_iter(),
+            follow_dagpb_data: true,
         }
     }
 }
@@ -90,7 +94,13 @@ impl TryFrom<&str> for IpfsPath {
 
         let root = Some(Cid::try_from(root).map_err(PathError::InvalidCid)?);
 
-        Ok(IpfsPath { root, path })
+        let follow_dagpb_data = true;
+
+        Ok(IpfsPath {
+            root,
+            path,
+            follow_dagpb_data,
+        })
     }
 }
 
@@ -99,6 +109,14 @@ impl IpfsPath {
         self.root.take()
     }
 
+    pub fn set_follow_dagpb_data(&mut self, follow: bool) {
+        self.follow_dagpb_data = follow;
+    }
+
+    pub fn follow_dagpb_data(&self) -> bool {
+        self.follow_dagpb_data
+    }
+
     pub fn resolve(&mut self, ipld: Ipld) -> Result<WalkSuccess, WalkFailed> {
         let key = match self.next() {
             Some(key) => key,