Use native ipfs types and remove hand-written ones (#243)

* Use native ipfs types and remove hand-written ones * Update all dependencies to their latest version * Fix nix syntax * Niv update & configure yarn with nodejs 16 * Fix Blob being undefined in nodejs * Move ipfs-core into devDependencies
oddsdk · Jun 4, 2021 · abe1b36 · abe1b36
1 parent d40e0af
commit abe1b36
Show file tree

Hide file tree

Showing 19 changed files with 2,493 additions and 3,343 deletions.
diff --git a/nix/sources.json b/nix/sources.json
@@ -5,10 +5,10 @@
         "homepage": "https://github.com/nmattia/niv",
         "owner": "nmattia",
         "repo": "niv",
-        "rev": "af958e8057f345ee1aca714c1247ef3ba1c15f5e",
-        "sha256": "1qjavxabbrsh73yck5dcq8jggvh3r2jkbr6b5nlz5d9yrqm9255n",
+        "rev": "1819632b5823e0527da28ad82fecd6be5136c1e9",
+        "sha256": "08jz17756qchq0zrqmapcm33nr4ms9f630mycc06i6zkfwl5yh5i",
         "type": "tarball",
-        "url": "https://github.com/nmattia/niv/archive/af958e8057f345ee1aca714c1247ef3ba1c15f5e.tar.gz",
+        "url": "https://github.com/nmattia/niv/archive/1819632b5823e0527da28ad82fecd6be5136c1e9.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixpkgs": {
@@ -29,10 +29,10 @@
         "homepage": null,
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "0a5f5bab0e08e968ef25cff393312aa51a3512cf",
-        "sha256": "161xisa5a7wi2h5hs0p1w7zbf095w21cs0wp0524kp7p6cpvvfyn",
+        "rev": "540dccb2aeaffa9dc69bfdc41c55abd7ccc6baa3",
+        "sha256": "1j58m811w7xxjncf36hqcjqsfj979hkfcwx9wcrm3g3zbayavapg",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/0a5f5bab0e08e968ef25cff393312aa51a3512cf.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/540dccb2aeaffa9dc69bfdc41c55abd7ccc6baa3.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }
diff --git a/nix/sources.nix b/nix/sources.nix
@@ -6,25 +6,33 @@ let
   # The fetchers. fetch_<type> fetches specs of type <type>.
   #
 
-  fetch_file = pkgs: spec:
-    if spec.builtin or true then
-      builtins_fetchurl { inherit (spec) url sha256; }
-    else
-      pkgs.fetchurl { inherit (spec) url sha256; };
+  fetch_file = pkgs: name: spec:
+    let
+      name' = sanitizeName name + "-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
+      else
+        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
 
   fetch_tarball = pkgs: name: spec:
     let
-      ok = str: ! builtins.isNull (builtins.match "[a-zA-Z0-9+-._?=]" str);
-      # sanitize the name, though nix will still fail if name starts with period
-      name' = stringAsChars (x: if ! ok x then "-" else x) "${name}-src";
+      name' = sanitizeName name + "-src";
     in
       if spec.builtin or true then
         builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
       else
         pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
 
-  fetch_git = spec:
-    builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
+  fetch_git = name: spec:
+    let
+      ref =
+        if spec ? ref then spec.ref else
+          if spec ? branch then "refs/heads/${spec.branch}" else
+            if spec ? tag then "refs/tags/${spec.tag}" else
+              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
+    in
+      builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
 
   fetch_local = spec: spec.path;
 
@@ -40,11 +48,21 @@ let
   # Various helpers
   #
 
+  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
+  sanitizeName = name:
+    (
+      concatMapStrings (s: if builtins.isList s then "-" else s)
+        (
+          builtins.split "[^[:alnum:]+._?=-]+"
+            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
+        )
+    );
+
   # The set of packages used when specs are fetched using non-builtins.
-  mkPkgs = sources:
+  mkPkgs = sources: system:
     let
       sourcesNixpkgs =
-        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {};
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
       hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
       hasThisAsNixpkgsPath = <nixpkgs> == ./.;
     in
@@ -64,9 +82,9 @@ let
 
     if ! builtins.hasAttr "type" spec then
       abort "ERROR: niv spec ${name} does not have a 'type' attribute"
-    else if spec.type == "file" then fetch_file pkgs spec
+    else if spec.type == "file" then fetch_file pkgs name spec
     else if spec.type == "tarball" then fetch_tarball pkgs name spec
-    else if spec.type == "git" then fetch_git spec
+    else if spec.type == "git" then fetch_git name spec
     else if spec.type == "local" then fetch_local spec
     else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
     else if spec.type == "builtin-url" then fetch_builtin-url name
@@ -80,7 +98,10 @@ let
       saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
       ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
     in
-      if ersatz == "" then drv else ersatz;
+      if ersatz == "" then drv else
+        # this turns the string into an actual Nix path (for both absolute and
+        # relative paths)
+        if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
 
   # Ports of functions for older nix versions
 
@@ -98,25 +119,29 @@ let
 
   # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
   stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
+  concatMapStrings = f: list: concatStrings (map f list);
   concatStrings = builtins.concatStringsSep "";
 
+  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
+  optionalAttrs = cond: as: if cond then as else {};
+
   # fetchTarball version that is compatible between all the versions of Nix
-  builtins_fetchTarball = { url, name, sha256 }@attrs:
+  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchTarball;
     in
       if lessThan nixVersion "1.12" then
-        fetchTarball { inherit name url; }
+        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
       else
         fetchTarball attrs;
 
   # fetchurl version that is compatible between all the versions of Nix
-  builtins_fetchurl = { url, sha256 }@attrs:
+  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchurl;
     in
       if lessThan nixVersion "1.12" then
-        fetchurl { inherit url; }
+        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
       else
         fetchurl attrs;
 
@@ -135,7 +160,8 @@ let
   mkConfig =
     { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
     , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
-    , pkgs ? mkPkgs sources
+    , system ? builtins.currentSystem
+    , pkgs ? mkPkgs sources system
     }: rec {
       # The sources, i.e. the attribute set of spec name to spec
       inherit sources;

diff --git a/package.json b/package.json
@@ -58,64 +58,65 @@
     "singleQuote": true
   },
   "devDependencies": {
-    "@babel/core": "^7.12.10",
-    "@babel/preset-env": "^7.12.7",
-    "@babel/preset-typescript": "^7.12.7",
+    "@babel/core": "^7.14.3",
+    "@babel/preset-env": "^7.14.2",
+    "@babel/preset-typescript": "^7.13.0",
     "@ipld/car": "https://github.com/matheus23/ipld-car-jest-fix#28fa48a8d0e701ddaf6e4785d0d9f08735b67bc8",
-    "@rollup/plugin-babel": "^5.2.2",
-    "@rollup/plugin-commonjs": "^17.0.0",
+    "@rollup/plugin-babel": "^5.3.0",
+    "@rollup/plugin-commonjs": "^19.0.0",
     "@rollup/plugin-inject": "^4.0.2",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^11.1.0",
-    "@types/jest": "^26.0.20",
+    "@rollup/plugin-node-resolve": "^13.0.0",
+    "@types/jest": "^26.0.23",
     "@types/jest-environment-puppeteer": "^4.4.1",
-    "@types/node": "^13.7.4",
+    "@types/node": "^15.6.1",
     "@types/throttle-debounce": "^2.1.0",
-    "@typescript-eslint/eslint-plugin": "^4.24.0",
-    "@typescript-eslint/parser": "^4.24.0",
-    "babel-jest": "^26.6.3",
+    "@typescript-eslint/eslint-plugin": "^4.25.0",
+    "@typescript-eslint/parser": "^4.25.0",
+    "babel-jest": "^27.0.1",
     "braces": "^3.0.2",
-    "eslint": "^7.26.0",
+    "eslint": "^7.27.0",
     "fast-check": "^2.14.0",
     "interface-datastore": "^4.0.1",
+    "ipfs-core": "^0.7.0",
+    "ipfs-core-types": "^0.5.0",
     "ipfs-repo": "^9.1.6",
-    "jest": "^26.6.3",
-    "jest-config": "^26.6.3",
-    "jest-puppeteer": "^5.0.3",
-    "lint-staged": "^10.5.3",
+    "jest": "^27.0.1",
+    "jest-config": "^27.0.1",
+    "jest-puppeteer": "^5.0.4",
+    "lint-staged": "^11.0.0",
     "multihashing-async": "^2.1.2",
-    "prettier": "^1.19.1",
+    "prettier": "^2.3.0",
     "prompt": "^1.1.0",
     "puppeteer": "^9.1.1",
-    "replace-in-file": "^5.0.2",
+    "replace-in-file": "^6.2.0",
     "rimraf": "^3.0.2",
-    "rollup": "^2.37.1",
+    "rollup": "^2.50.3",
     "rollup-plugin-gzip": "^2.5.0",
     "rollup-plugin-node-polyfills": "^0.2.1",
     "rollup-plugin-sourcemaps": "^0.6.3",
     "rollup-plugin-terser": "^7.0.2",
-    "rollup-plugin-typescript2": "^0.29.0",
-    "tslib": "^2.1.0",
-    "typedoc": "^0.20.23",
-    "typescript": "^4.1.3",
+    "rollup-plugin-typescript2": "^0.30.0",
+    "tslib": "^2.2.0",
+    "typedoc": "^0.20.36",
+    "typescript": "^4.3.2",
     "typescript-documentation": "^2.0.0",
     "yarn": "^1.22.4"
   },
   "dependencies": {
     "base58-universal": "https://github.com/digitalbazaar/base58-universal#de970560f005de0f7054723c35ef6e0ff4b328b7",
     "borc": "^3.0.0",
     "buffer": "^6.0.3",
-    "cids": "^1.1.5",
-    "fission-bloom-filters": "1.6.0",
-    "ipfs-core": "^0.6.1",
+    "cids": "^1.1.6",
+    "fission-bloom-filters": "1.7.0",
     "ipfs-message-port-client": "https://ipfs.runfission.com/ipfs/bafybeigx6q4aezve7my76s5vvfuiinbxtepapqvmjf2jbgrozrut6cjape/p/ipfs-message-port-client.tar.gz",
     "ipfs-message-port-protocol": "https://ipfs.runfission.com/ipfs/bafybeigx6q4aezve7my76s5vvfuiinbxtepapqvmjf2jbgrozrut6cjape/p/ipfs-message-port-protocol.tar.gz",
-    "ipld-dag-pb": "^0.20.0",
+    "ipld-dag-pb": "^0.22.2",
     "keystore-idb": "0.14.1",
     "localforage": "^1.9.0",
     "make-error": "^1.3.6",
-    "noble-ed25519": "^1.0.4",
-    "throttle-debounce": "^2.2.1"
+    "noble-ed25519": "^1.2.2",
+    "throttle-debounce": "^3.0.1"
   },
   "resolutions": {
     "**/ipfs-message-port-protocol": "https://ipfs.runfission.com/ipfs/bafybeigx6q4aezve7my76s5vvfuiinbxtepapqvmjf2jbgrozrut6cjape/p/ipfs-message-port-protocol.tar.gz"

diff --git a/shell.nix b/shell.nix
@@ -2,17 +2,18 @@ let
   sources  = import ./nix/sources.nix;
   pkgs     = import sources.nixpkgs  {};
   unstable = import sources.unstable {};
+  # https://github.com/NixOS/nixpkgs/issues/53820
+  yarn = unstable.yarn.override { nodejs = unstable.nodejs-16_x; };
 in
 
 pkgs.mkShell {
   buildInputs = [
-    # https://github.com/NixOS/nixpkgs/issues/53820#issuecomment-617973476
-    (unstable.yarn.override { nodejs = null; })
-    unstable.nodejs-15_x
+    yarn
+    unstable.nodejs-16_x
     unstable.niv
   ];
 
   shellHook = ''
-    ${unstable.yarn}/bin/yarn install
+    ${yarn}/bin/yarn install
   '';
 }
diff --git a/src/@types/index.d.ts b/src/@types/index.d.ts
@@ -2,5 +2,4 @@ declare module 'base58-universal' {
     export function encode(input: Uint8Array, maxline?: number): string
     export function decode(input: string): Uint8Array
 }
-declare module 'ipld-dag-pb'
 declare module 'borc'
diff --git a/src/fs/bare/file.ts b/src/fs/bare/file.ts
@@ -25,7 +25,7 @@ export class BareFile extends BaseFile {
   }
 
   async putDetailed(): Promise<AddResult> {
-    return protocol.basic.putFile(this.content)
+    return protocol.basic.putFile(await protocol.pub.normalizeFileContent(this.content))
   }
 }
 

diff --git a/src/fs/link.ts b/src/fs/link.ts
@@ -1,15 +1,16 @@
-import dagPB from 'ipld-dag-pb'
+import dagPB, { DAGLink } from 'ipld-dag-pb'
+import type { IPFSEntry } from 'ipfs-core-types/src/root'
 
-import { DAGLink, UnixFSFile } from '../ipfs'
 import { Link, SimpleLink } from './types'
+import { mtimeFromMs } from './metadata'
 
 
 export const toDAGLink = (link: SimpleLink): DAGLink => {
   const { name, cid, size } = link
   return new dagPB.DAGLink(name, size, cid)
 }
 
-export const fromFSFile = (fsObj: UnixFSFile): Link => {
+export const fromFSFile = (fsObj: IPFSEntry): Link => {
   const { name = '', cid, size, mtime, type } = fsObj
   return {
     name,
@@ -33,7 +34,7 @@ export const make = (name: string, cid: string, isFile: boolean, size: number):
     cid,
     size,
     isFile,
-    mtime: Date.now()
+    mtime: mtimeFromMs(Date.now())
   }
 }
 

diff --git a/src/fs/metadata.ts b/src/fs/metadata.ts
@@ -1,3 +1,4 @@
+import type { Mtime } from 'ipfs-unixfs'
 import * as semver from './semver'
 import { SemVer } from './semver'
 
@@ -45,3 +46,11 @@ export const updateMtime = (metadata: Metadata): Metadata => ({
     mtime: Date.now()
   }
 })
+
+export function mtimeFromMs(ms: number): Mtime {
+  const secs = Math.floor(ms / 1000)
+  return {
+    secs: secs,
+    nsecs: (ms - (secs * 1000)) * 1000
+  }
+}
diff --git a/src/fs/protocol/basic.ts b/src/fs/protocol/basic.ts
@@ -1,22 +1,22 @@
 /** @internal */
+import type { ImportCandidate } from 'ipfs-core-types/src/utils'
 
-/** @internal */
 import * as ipfs from '../../ipfs'
 import { CID, FileContent, AddResult } from '../../ipfs'
 
 import { SimpleLinks, Links } from '../types'
 import * as link from '../link'
 
 
-export const getFile = async (cid: CID): Promise<FileContent> => {
+export const getFile = async (cid: CID): Promise<Uint8Array> => {
   return ipfs.catBuf(cid)
 }
 
 export const getEncryptedFile = async (cid: CID, key: string): Promise<FileContent> => {
   return ipfs.encoded.catAndDecode(cid, key) as Promise<FileContent>
 }
 
-export const putFile = async (content: FileContent): Promise<AddResult> => {
+export const putFile = async (content: ImportCandidate): Promise<AddResult> => {
   return ipfs.add(content)
 }
 

diff --git a/src/fs/protocol/private/mmpt.test.ts b/src/fs/protocol/private/mmpt.test.ts
@@ -16,16 +16,14 @@ function encode(str: string): Uint8Array {
 
 let ipfs: IPFS | null = null
 
-beforeAll(async done => {
+beforeAll(async () => {
   ipfs = await createInMemoryIPFS()
   ipfsConfig.set(ipfs)
-  done()
 })
 
-afterAll(async done => {
+afterAll(async () => {
   if (ipfs == null) return
   await ipfs.stop()
-  done()
 })
 
 /*
-Original file line number
+Diff line change
@@ Expand Up / @@ -25,7 +25,7 @@ export class BareFile extends BaseFile { @@
       }
       async putDetailed(): Promise<AddResult> {
-        return protocol.basic.putFile(this.content)
+        return protocol.basic.putFile(await protocol.pub.normalizeFileContent(this.content))
       }
     }
@@ Expand Down @@