feat: support UnixFSv1.5 metadata (#1186)

* feat: support UnixFSv1.5 metadata

* fix: expose new mfs functions

* refactor: send mtime and mode as headers instead of message parts

* fix: include headers for directories

* chore: update ipfs utils dep version

* chore: update ipfs-utils dep

* fix: stringify mode in browser

* test: add tests for unixfs metadata

* fix: fix up tests etc for optional mtime

package.json

@@ -54,7 +54,7 @@
  "explain-error": "^1.0.4",
  "form-data": "^3.0.0",
  "ipfs-block": "~0.8.1",
- "ipfs-utils": "^0.4.0",
+ "ipfs-utils": "^0.4.2",
  "ipld-dag-cbor": "~0.15.0",
  "ipld-dag-pb": "^0.18.1",
  "ipld-raw": "^4.0.0",
@@ -84,7 +84,7 @@
  "cross-env": "^6.0.0",
  "detect-node": "^2.0.4",
  "go-ipfs-dep": "^0.4.22",
- "interface-ipfs-core": "~0.125.0",
+ "interface-ipfs-core": "^0.126.0",
  "ipfsd-ctl": "^1.0.0",
  "ndjson": "^1.5.0",
  "nock": "^11.4.0",

src/add/form-data.browser.js

@@ -2,13 +2,29 @@
 /* eslint-env browser */
 
 const normaliseInput = require('ipfs-utils/src/files/normalise-input')
+const mtimeToObject = require('../lib/mtime-to-object')
 
 exports.toFormData = async input => {
  const files = normaliseInput(input)
  const formData = new FormData()
  let i = 0
 
  for await (const file of files) {
+ const headers = {}
+
+ if (file.mtime !== undefined && file.mtime !== null) {
+ const mtime = mtimeToObject(file.mtime)
+
+ if (mtime) {
+ headers.mtime = mtime.secs
+ headers['mtime-nsecs'] = mtime.nsecs
+ }
+ }
+
+ if (file.mode !== undefined && file.mode !== null) {
+ headers.mode = file.mode.toString(8).padStart(4, '0')
+ }
+
  if (file.content) {
  // In the browser there's _currently_ no streaming upload, buffer up our
  // async iterator chunks and append a big Blob :(
@@ -18,9 +34,13 @@ exports.toFormData = async input => {
  bufs.push(chunk)
  }
 
- formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), encodeURIComponent(file.path))
+ formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), encodeURIComponent(file.path), {
+ header: headers
+ })
  } else {
- formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), encodeURIComponent(file.path))
+ formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), encodeURIComponent(file.path), {
+ header: headers
+ })
  }
 
  i++

src/add/form-data.js

@@ -5,13 +5,29 @@ const { Buffer } = require('buffer')
 const toStream = require('it-to-stream')
 const normaliseInput = require('ipfs-utils/src/files/normalise-input')
 const { isElectronRenderer } = require('ipfs-utils/src/env')
+const mtimeToObject = require('../lib/mtime-to-object')
 
 exports.toFormData = async input => {
  const files = normaliseInput(input)
  const formData = new FormData()
  let i = 0
 
  for await (const file of files) {
+ const headers = {}
+
+ if (file.mtime !== undefined && file.mtime !== null) {
+ const mtime = mtimeToObject(file.mtime)
+
+ if (mtime) {
+ headers.mtime = mtime.secs
+ headers['mtime-nsecs'] = mtime.nsecs
+ }
+ }
+
+ if (file.mode !== undefined && file.mode !== null) {
+ headers.mode = file.mode.toString(8).padStart(4, '0')
+ }
+
  if (file.content) {
  // In Node.js, FormData can be passed a stream so no need to buffer
  formData.append(
@@ -26,13 +42,15 @@ exports.toFormData = async input => {
  {
  filepath: encodeURIComponent(file.path),
  contentType: 'application/octet-stream',
- knownLength: file.content.length // Send Content-Length header if known
+ knownLength: file.content.length, // Send Content-Length header if known
+ header: headers
  }
  )
  } else {
  formData.append(`dir-${i}`, Buffer.alloc(0), {
  filepath: encodeURIComponent(file.path),
- contentType: 'application/x-directory'
+ contentType: 'application/x-directory',
+ header: headers
  })
  }
 

src/add/index.js

@@ -52,6 +52,16 @@ module.exports = configure(({ ky }) => {
  }
 })
 
-function toCoreInterface ({ name, hash, size }) {
- return { path: name, hash, size: parseInt(size) }
+function toCoreInterface ({ name, hash, size, mode, mtime }) {
+ const output = {
+ path: name,
+ hash,
+ size: parseInt(size)
+ }
+
+ if (mode !== undefined) {
+ output.mode = parseInt(mode, 8)
+ }
+
+ return output
 }

src/files/chmod.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const configure = require('../lib/configure')
+const modeToString = require('../lib/mode-to-string')
+
+module.exports = configure(({ ky }) => {
+ return function chmod (path, mode, options) {
+ options = options || {}
+
+ const searchParams = new URLSearchParams(options.searchParams)
+ searchParams.append('arg', path)
+ searchParams.append('mode', modeToString(mode))
+ if (options.format) searchParams.set('format', options.format)
+ if (options.flush != null) searchParams.set('flush', options.flush)
+ if (options.hashAlg) searchParams.set('hash', options.hashAlg)
+ if (options.parents != null) searchParams.set('parents', options.parents)
+
+ return ky.post('files/chmod', {
+ timeout: options.timeout,
+ signal: options.signal,
+ headers: options.headers,
+ searchParams
+ }).text()
+ }
+})

src/files/index.js

@@ -8,6 +8,7 @@ module.exports = config => {
  const read = require('./read')(config)
 
  return {
+ chmod: callbackify.variadic(require('./chmod')(config)),
  cp: callbackify.variadic(require('./cp')(config)),
  mkdir: callbackify.variadic(require('./mkdir')(config)),
  flush: callbackify.variadic(require('./flush')(config)),
@@ -19,6 +20,7 @@ module.exports = config => {
  read: callbackify.variadic(concatify(read)),
  readReadableStream: streamify.readable(read),
  readPullStream: pullify.source(read),
+ touch: callbackify.variadic(require('./touch')(config)),
  write: callbackify.variadic(require('./write')(config)),
  mv: callbackify.variadic(require('./mv')(config))
  }

src/files/ls.js

@@ -4,7 +4,7 @@ const CID = require('cids')
 const ndjson = require('iterable-ndjson')
 const toIterable = require('../lib/stream-to-iterable')
 const configure = require('../lib/configure')
-const toCamel = require('../lib/object-to-camel')
+const toCamelWithMetadata = require('../lib/object-to-camel-with-metadata')
 
 module.exports = configure(({ ky }) => {
  return async function * ls (path, options) {
@@ -32,11 +32,12 @@ module.exports = configure(({ ky }) => {
  // go-ipfs does not yet support the "stream" option
  if ('Entries' in result) {
  for (const entry of result.Entries || []) {
- yield toCamel(entry)
+ yield toCamelWithMetadata(entry)
  }
  return
  }
- yield toCamel(result)
+
+ yield toCamelWithMetadata(result)
  }
  }
 })

src/files/mkdir.js

@@ -1,10 +1,13 @@
 'use strict'
 
 const configure = require('../lib/configure')
+const modeToString = require('../lib/mode-to-string')
+const mtimeToObject = require('../lib/mtime-to-object')
 
 module.exports = configure(({ ky }) => {
  return (path, options) => {
  options = options || {}
+ const mtime = mtimeToObject(options.mtime)
 
  const searchParams = new URLSearchParams(options.searchParams)
  searchParams.append('arg', path)
@@ -13,6 +16,14 @@ module.exports = configure(({ ky }) => {
  if (options.flush != null) searchParams.set('flush', options.flush)
  if (options.hashAlg) searchParams.set('hash', options.hashAlg)
  if (options.parents != null) searchParams.set('parents', options.parents)
+ if (mtime) {
+ searchParams.set('mtime', mtime.secs)
+
+ if (mtime.nsecs != null) {
+ searchParams.set('mtimeNsecs', mtime.nsecs)
+ }
+ }
+ if (options.mode != null) searchParams.set('mode', modeToString(options.mode))
 
  return ky.post('files/mkdir', {
  timeout: options.timeout,

src/files/stat.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const configure = require('../lib/configure')
-const toCamel = require('../lib/object-to-camel')
+const toCamelWithMetadata = require('../lib/object-to-camel-with-metadata')
 
 module.exports = configure(({ ky }) => {
  return async (path, options) => {
@@ -27,6 +27,7 @@ module.exports = configure(({ ky }) => {
  }).json()
 
  res.WithLocality = res.WithLocality || false
- return toCamel(res)
+
+ return toCamelWithMetadata(res)
  }
 })

src/files/touch.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const configure = require('../lib/configure')
+const mtimeToObject = require('../lib/mtime-to-object')
+
+module.exports = configure(({ ky }) => {
+ return function touch (path, options) {
+ options = options || {}
+ const mtime = mtimeToObject(options.mtime)
+
+ const searchParams = new URLSearchParams(options.searchParams)
+ searchParams.append('arg', path)
+ if (mtime) {
+ searchParams.set('mtime', mtime.secs)
+ searchParams.set('mtimeNsecs', mtime.nsecs)
+ }
+ if (options.format) searchParams.set('format', options.format)
+ if (options.flush != null) searchParams.set('flush', options.flush)
+ if (options.hashAlg) searchParams.set('hash', options.hashAlg)
+ if (options.parents != null) searchParams.set('parents', options.parents)
+
+ return ky.post('files/touch', {
+ timeout: options.timeout,
+ signal: options.signal,
+ headers: options.headers,
+ searchParams
+ }).text()
+ }
+})

src/files/write.js

@@ -2,10 +2,13 @@
 
 const configure = require('../lib/configure')
 const toFormData = require('../lib/buffer-to-form-data')
+const modeToString = require('../lib/mode-to-string')
+const mtimeToObject = require('../lib/mtime-to-object')
 
 module.exports = configure(({ ky }) => {
  return async (path, input, options) => {
  options = options || {}
+ const mtime = mtimeToObject(options.mtime)
 
  const searchParams = new URLSearchParams(options.searchParams)
  searchParams.set('arg', path)
@@ -18,13 +21,24 @@ module.exports = configure(({ ky }) => {
  if (options.parents != null) searchParams.set('parents', options.parents)
  if (options.rawLeaves != null) searchParams.set('raw-leaves', options.rawLeaves)
  if (options.truncate != null) searchParams.set('truncate', options.truncate)
+ if (mtime) {
+ searchParams.set('mtime', mtime.secs)
+
+ if (mtime.nsecs != null) {
+ searchParams.set('mtimeNsecs', mtime.nsecs)
+ }
+ }
 
  const res = await ky.post('files/write', {
  timeout: options.timeout,
  signal: options.signal,
  headers: options.headers,
  searchParams,
- body: toFormData(input) // TODO: support inputs other than buffer as per spec
+ body: toFormData(input, {
+ mode: options.mode != null ? modeToString(options.mode) : undefined,
+ mtime: mtime ? mtime.secs : undefined,
+ mtimeNsecs: mtime ? mtime.nsecs : undefined
+ }) // TODO: support inputs other than buffer as per spec
  })
 
  return res.text()

src/lib/buffer-to-form-data.js

@@ -3,9 +3,25 @@
 const FormData = require('form-data')
 const { isElectronRenderer } = require('ipfs-utils/src/env')
 
-module.exports = buf => {
+module.exports = (buf, { mode, mtime, mtimeNsecs } = {}) => {
+ const headers = {}
+
+ if (mode != null) {
+ headers.mode = mode
+ }
+
+ if (mtime != null) {
+ headers.mtime = mtime
+
+ if (mtimeNsecs != null) {
+ headers['mtime-nsecs'] = mtimeNsecs
+ }
+ }
+
  const formData = new FormData()
- formData.append('file', buf)
+ formData.append('file', buf, {
+ header: headers
+ })
  return formData
 }
 

src/lib/mode-to-string.js

@@ -0,0 +1,13 @@
+'use strict'
+
+module.exports = (mode) => {
+ if (mode === undefined || mode === null) {
+ return undefined
+ }
+
+ if (typeof mode === 'string' || mode instanceof String) {
+ return mode
+ }
+
+ return mode.toString(8).padStart(4, '0')
+}