fix: add files and folders (#931)

The fix herewas to normalise and fix how we calculated:

- the paths within a dropped folder (FF was getting doubled
wrapped in the root dir)
- the number of dirs that would be created for a given tree,
so we can correctly calculate if we got the right number of
items back in the call to ipfs.add
...and removing the unused wrapping dir when adding.


License: MIT
Signed-off-by: Oli Evans <oli@tableflip.io>

src/bundles/files.js

@@ -1,6 +1,7 @@
 import { join, dirname } from 'path'
 import { createSelector } from 'redux-bundler'
 import { getDownloadLink, getShareableLink, filesToStreams } from '../lib/files'
+import countDirs from '../lib/count-dirs'
 import ms from 'milliseconds'
 
 const isMac = navigator.userAgent.indexOf('Mac') !== -1
@@ -229,6 +230,14 @@ export default (opts = {}) => {
     doFilesWrite: make(actions.WRITE, async (ipfs, root, rawFiles, id, { dispatch }) => {
       const { streams, totalSize } = await filesToStreams(rawFiles)
 
+      // Normalise all paths to be relative. Dropped files come as absolute,
+      // those added by the file input come as relative paths, so normalise them.
+      streams.forEach(s => {
+        if (s.path[0] === '/') {
+          s.path = s.path.slice(1)
+        }
+      })
+
       const updateProgress = (sent) => {
         dispatch({ type: 'FILES_WRITE_UPDATED', payload: { id: id, progress: sent / totalSize * 100 } })
       }
@@ -237,11 +246,15 @@ export default (opts = {}) => {
 
       const res = await ipfs.add(streams, {
         pin: false,
-        wrapWithDirectory: true,
+        wrapWithDirectory: false,
         progress: updateProgress
       })
 
-      if (res.length !== streams.length + 2) {
+      const numberOfFiles = streams.length
+      const numberOfDirs = countDirs(streams)
+      const expectedResponseCount = numberOfFiles + numberOfDirs
+
+      if (res.length !== expectedResponseCount) {
         // See https://github.com/ipfs/js-ipfs-api/issues/797
         throw Object.assign(new Error(`API returned a partial response.`), { code: 'ERR_API_RESPONSE' })
       }
@@ -255,6 +268,7 @@ export default (opts = {}) => {
           try {
             await ipfs.files.cp([src, dst])
           } catch (err) {
+            console.log(err, { root, path, src, dst })
             throw Object.assign(new Error(`Folder already exists.`), { code: 'ERR_FOLDER_EXISTS' })
           }
         }

src/lib/count-dirs.js

@@ -0,0 +1,44 @@
+import { dirname } from 'path'
+
+/**
+* countDirs: find all the dirs that will be created from a list of paths.
+*
+* files is an array of file objects as passed to ipfs.added
+* The root dir is counted, and All entries are assumed to be file paths,
+* `/foo` is assumed to be a file `foo` with no extention in the root dir,
+* which would be counted as 1 unigue dir by this function.
+*
+* ```js
+* files = [
+*   { path: '/foo/bar/foo.txt', ... }
+*   { path: '/foo/bar/odd', ... }
+* ]
+* countDirs(files) === 3
+* // ['/', '/foo', '/foo/bar']
+* ```
+*
+* We need to calculat how many directories are in the tree.
+*
+* See: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#add
+*/
+function countDirs (files) {
+  if (!files || !files.length) return 0
+  const paths = files.map(f => f.path)
+    .filter(p => !!p)
+
+  // [ /foo/bar, /foo/other, /foo/zoom, /aaa/other ]
+  const directories = new Set()
+  paths.forEach(path => findUniqueDirectories(path, directories))
+  return directories.size
+}
+
+function findUniqueDirectories (path, res = new Set()) {
+  if (!path) return res
+  const name = dirname(path)
+  if (name === '.') return res
+  res.add(name)
+  if (name === '/') return res
+  return findUniqueDirectories(name, res)
+}
+
+export default countDirs

src/lib/count-dirs.test.js

@@ -0,0 +1,43 @@
+/* global it, expect */
+import countDirs from './count-dirs'
+
+it('should return 1 for the root dir', () => {
+  expect(countDirs([{ path: '/' }])).toBe(1)
+})
+
+it('should return 2 for the root dir with a sub dir', () => {
+  expect(countDirs([{ path: '/foo/x' }])).toBe(2)
+})
+
+it('should return 1 for the root dir with an extentionless file', () => {
+  expect(countDirs([{ path: '/foo' }])).toBe(1)
+})
+
+it('should return 0 for if files is empty', () => {
+  expect(countDirs([])).toBe(0)
+})
+
+it('should return 0 for if files is missing', () => {
+  expect(countDirs()).toBe(0)
+})
+
+it('should return 0 for a file name', () => {
+  expect(countDirs([{ path: 'Master layout FINAL v18(2).pdf' }])).toBe(0)
+})
+
+it('should deal with relative paths', () => {
+  expect(countDirs([{ path: 'home/www/index.html' }])).toBe(2)
+})
+
+it('should count the unique dirs in a list of file objects', () => {
+  let files = [
+    { path: '/foo/bar/foo.txt' },
+    { path: '/foo/bar/odd.txt' },
+    { path: '/foo/other/odd.txt' },
+    { path: '/foo/zoom/x.txt' },
+    { path: '/aaa/other/y.txt' },
+    { path: '/aaa/bar/y.txt' }
+  ]
+
+  expect(countDirs(files)).toBe(8)
+})

src/lib/dnd-backend.js

@@ -23,9 +23,9 @@ const readEntries = (reader) => new Promise((resolve, reject) => {
 async function scanFiles (item, root = '') {
   if (!item.isDirectory) {
     const file = await getFileFromEntry(item)
-
+    const path = item.fullPath
     return [{
-      path: join(root, file.webkitRelativePath || file.name),
+      path,
       content: fileReader(file),
       size: file.size
     }]