Skip to content

Commit

Permalink
Fixes batches in Dropbox
Browse files Browse the repository at this point in the history
* Fixes write batch in Dropbox
* Batch correctly overwrite existing files
* Batch now rejects the promise if one writing action failed
* Fixes delete batch
* Uses Buffer for Dropbox batch uploads

Closes #131
Closes #130
  • Loading branch information
JbIPS authored Nov 26, 2017
1 parent b0e7180 commit f1cb72e
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 13 deletions.
13 changes: 10 additions & 3 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,20 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/).

## [Unreleased]
### Fixed
- [Dropbox] Batch correctly overwrite existing files (https://github.com/silexlabs/unifile/issues/131)
- [Dropbox] Batch now correctly rejects the promise if one action failed (https://github.com/silexlabs/unifile/issues/131)
- [Dropbox] Batch upload uses `Buffer` for file content and supports UTF-8 (https://github.com/silexlabs/unifile/issues/130)

## [2.0.0] - 2017-11-25
### Changed
- GitHub batch fixes and optimization
- Code factorization
- Remove parameters mutations
- `.readFile()` now always return a `Buffer`
- [Dropbox] Retrieve account when providing only the token
- [Dropbox] Normalize errors (as for #103)
- [Dropbox] Normalize errors (https://github.com/silexlabs/unifile/issues/103)

### Added
- Tools, index and FS are 100% covered
Expand All @@ -19,7 +25,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p

### Fixed
- In SFTP, directory type is now set to 'application/directory'
- [Dropbox] Fixes batch upload (Closes ##114)
- [Dropbox] Fixes batch upload (https://github.com/silexlabs/unifile/issues/114)

### Removed
- WebDAV connector is now a separate plugin
Expand Down Expand Up @@ -51,6 +57,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
### Changed
- Total rework of the philosphy

[2.0.0]: https://github.com/silexlabs/unifile/compare/v1.2.0...v
[Unreleased]: https://github.com/silexlabs/unifile/compare/v2.0.0...HEAD
[2.0.0]: https://github.com/silexlabs/unifile/compare/v1.2.0...v2.0.0
[1.2.0]: https://github.com/silexlabs/unifile/compare/v1.1.0...v1.2.0
[1.1.0]: https://github.com/silexlabs/unifile/compare/v1.0.0...v1.1.0
34 changes: 29 additions & 5 deletions lib/unifile-dropbox.js
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,18 @@ function checkBatchEnd(session, result, checkRoute, jobId) {
})
.then((result) => checkBatchEnd(session, result, checkRoute, newId));
case 'complete':
const failed = result.entries.reduce((memo, entry, index) => {
if(entry['.tag'] === 'failure') memo.push({entry, index});
return memo;
}, []);
if(failed.length > 0) {
const errors = failed.map(({entry, index}) => {
const failureTag = entry.failure['.tag'];
return `Could not complete action ${index}: ${failureTag + '/' + entry.failure[failureTag]['.tag']}`;
});
return Promise.reject(new UnifileError(
UnifileError.EIO, errors.join(', ')));
}
return Promise.resolve();
}
}
Expand Down Expand Up @@ -408,6 +420,7 @@ class DropboxConnector {
}

batch(session, actions, message) {
const writeMode = this.writeMode;
let actionsChain = Promise.resolve();

let uploadEntries = [];
Expand Down Expand Up @@ -444,10 +457,20 @@ class DropboxConnector {
function deleteBatch() {
if(deleteEntries.length === 0) return Promise.resolve();

const toDelete = deleteEntries.slice();
/*
Dropbox executes all the deletion at the same time,
so we remove all the descendant of a deleted folder beforehand
*/
const toDelete = deleteEntries.slice().sort((a, b) => a.path.length - b.path.length);
const deduplicated = [];
while(toDelete.length !== 0) {
if(!deduplicated.some(({path}) => toDelete[0].path.includes(path + '/'))) {
deduplicated.push(toDelete.shift());
} else toDelete.shift();
}
actionsChain = actionsChain.then(() => {
return callAPI(session, '/files/delete_batch', {
entries: toDelete
entries: deduplicated
})
.then((result) => checkBatchEnd(session, result, '/files/delete_batch/check'));
});
Expand All @@ -460,16 +483,17 @@ class DropboxConnector {
const toUpload = uploadEntries.slice();
actionsChain = actionsChain.then(() => {
return Promise.map(toUpload, (action) => {
return openUploadSession(session, action.content, true)
const bitContent = new Buffer(action.content);
return openUploadSession(session, bitContent, true)
.then((result) => {
return {
cursor: {
session_id: result.session_id,
offset: action.content.length
offset: bitContent.length
},
commit: {
path: makePathAbsolute(action.path),
mode: this.writeMode
mode: writeMode
}
};
});
Expand Down
63 changes: 58 additions & 5 deletions test/unifile-dropbox.js
Original file line number Diff line number Diff line change
Expand Up @@ -731,10 +731,11 @@ describe('DropboxConnector', function() {
describe('batch()', function() {
this.timeout(30000);
let connector;
const content = 'lorem ipsum';
const creation = [
{name: 'mkdir', path: 'tmp'},
{name: 'mkdir', path: 'tmp/test'},
{name: 'writeFile', path: 'tmp/test/a', content: 'aaa'},
{name: 'writeFile', path: 'tmp/test/a', content},
{name: 'mkdir', path: 'tmp/test/dir'},
{name: 'rename', path: 'tmp/test/a', destination: 'tmp/test/b'},
{name: 'mkdir', path: 'tmp2'},
Expand Down Expand Up @@ -769,14 +770,29 @@ describe('DropboxConnector', function() {
.should.be.rejectedWith('Write actions should have a content');
});

it('rejects the promise if an conflict happen', function() {
it('rejects the promise if a conflict happen', function() {
return connector.batch(session, [
{name: 'mkdir', path: 'tmp'},
{name: 'mkdir', path: 'tmp'}
]).should.be.rejectedWith('conflict')
.then(() => connector.rmdir(session, 'tmp'));
});

it('rejects the promise if a conflict happen and overwrite is not set', function() {
const path = 'tmp/indexFile';
const fileContent = 'html';
const noOverwriteConnector = new DropboxConnector(Object.assign({}, authConfig, {writeMode: 'add'}));
return noOverwriteConnector.writeFile(session, path, 'lorem')
.then(() => {
return noOverwriteConnector.batch(session, [{
name: 'writefile',
path: path,
content: fileContent
}]);
}).should.be.rejectedWith('Could not complete action 0: path/conflict')
.then(() => connector.rmdir(session, 'tmp'));
});

it('executes action in order', function() {
return connector.batch(session, creation)
.then(() => {
Expand All @@ -785,8 +801,8 @@ describe('DropboxConnector', function() {
expect(connector.readdir(session, 'tmp3')).to.be.fulfilled
]);
})
.then((content) => {
return expect(content[0].toString()).to.equal('aaa');
.then((results) => {
return expect(results[0].toString()).to.equal(content);
})
.then(() => connector.batch(session, destruction))
.then(() => {
Expand All @@ -797,12 +813,49 @@ describe('DropboxConnector', function() {
});
});

it('can write files with special chars', function() {
const path = 'tmp/specialFile';
const fileContent = 'Àà çéèîï';
return connector.batch(session, [{
name: 'writefile',
path: path,
content: fileContent
}])
.then(() => {
return connector.readFile(session, path);
})
.then((content) => {
return expect(content.toString()).to.equal(fileContent);
})
.then(() => connector.rmdir(session, 'tmp'));
});

it('can overwrite existing files', function() {
const path = 'tmp/indexFile';
const fileContent = 'html';
return connector.writeFile(session, path, 'lorem')
.then(() => {
return connector.batch(session, [{
name: 'writefile',
path: path,
content: fileContent
}]);
})
.then(() => {
return connector.readFile(session, path);
})
.then((content) => {
return expect(content.toString()).to.equal(fileContent);
})
.then(() => connector.rmdir(session, 'tmp'));
});

it('executes action in order and ignores unsupported ones', function() {
creation.unshift({name: 'createReadStream', path: 'unknown_file'});
creation.unshift({name: 'createReadStream', path: 'a/test/unknown_file'});
return connector.batch(session, creation)
.then(() => {
expect(connector.readFile(session, 'tmp/test/b')).to.become('aaa');
expect(connector.readFile(session, 'tmp/test/b')).to.become(content);
return connector.batch(session, destruction);
})
.then(() => {
Expand Down

0 comments on commit f1cb72e

Please sign in to comment.