diff --git a/lib/resources/odata.js b/lib/resources/odata.js
index 1a21adadc..608907d68 100644
--- a/lib/resources/odata.js
+++ b/lib/resources/odata.js
@@ -64,12 +64,11 @@ module.exports = (service, endpoint) => {
const options = QueryOptions.fromODataRequest(params, query);
return Promise.all([
Forms.getFields(form.def.id),
- Submissions.streamForExport(form.id, draft, undefined, options),
((params.table === 'Submissions') && options.hasPaging())
? Submissions.countByFormId(form.id, draft, options) : resolve(null)
])
- .then(([ fields, stream, count ]) =>
- json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count)));
+ .then(([ fields, count ]) => Submissions.streamForExport(form.id, draft, undefined, options)
+ .then((stream) => json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count))));
})));
};
diff --git a/lib/resources/submissions.js b/lib/resources/submissions.js
index 4232b1748..e06d55e7e 100644
--- a/lib/resources/submissions.js
+++ b/lib/resources/submissions.js
@@ -7,7 +7,7 @@
// including this file, may be copied, modified, propagated, or distributed
// except according to the terms contained in the LICENSE file.
-const { always, identity } = require('ramda');
+const { always, identity, call } = require('ramda');
const multer = require('multer');
const sanitize = require('sanitize-filename');
const { Blob, Form, Submission } = require('../model/frames');
@@ -269,20 +269,20 @@ module.exports = (service, endpoint) => {
const options = QueryOptions.fromSubmissionCsvRequest(query);
return Promise.all([
(options.deletedFields === true) ? Forms.getMergedFields(form.id) : Forms.getFields(form.def.id),
- Submissions.streamForExport(form.id, draft, keys, options),
(options.splitSelectMultiples !== true) ? null : Submissions.getSelectMultipleValuesForExport(form.id, draft, options),
- SubmissionAttachments.streamForExport(form.id, draft, keys, options),
- ClientAudits.streamForExport(form.id, draft, keys, options),
draft ? null : Audits.log(auth.actor, 'form.submission.export', form)
- ]).then(([ fields, rows, selectValues, attachments, clientAudits ]) => {
+ ]).then(([ fields, selectValues ]) => {
const filename = sanitize(form.xmlFormId);
response.append('Content-Disposition', contentDisposition(`${filename}.zip`));
response.append('Content-Type', 'application/zip');
return zipStreamFromParts(
// TODO: not 100% sure that these streams close right on crash.
- streamBriefcaseCsvs(rows, fields, form.xmlFormId, selectValues, decryptor, false, options),
- streamAttachments(attachments, decryptor),
- streamClientAudits(clientAudits, form, decryptor)
+ () => Submissions.streamForExport(form.id, draft, keys, options)
+ .then((rows) => streamBriefcaseCsvs(rows, fields, form.xmlFormId, selectValues, decryptor, false, options)),
+ () => SubmissionAttachments.streamForExport(form.id, draft, keys, options)
+ .then((attachments) => streamAttachments(attachments, decryptor)),
+ () => ClientAudits.streamForExport(form.id, draft, keys, options)
+ .then((clientAudits) => streamClientAudits(clientAudits, form, decryptor))
);
});
}));
@@ -295,18 +295,18 @@ module.exports = (service, endpoint) => {
const options = QueryOptions.fromSubmissionCsvRequest(query);
return Promise.all([
(options.deletedFields === true) ? Forms.getMergedFields(form.id) : Forms.getFields(form.def.id),
- Submissions.streamForExport(form.id, draft, Object.keys(passphrases), options),
(options.splitSelectMultiples !== true) ? null : Submissions.getSelectMultipleValuesForExport(form.id, draft, options),
Keys.getDecryptor(passphrases),
draft ? null : Audits.log(auth.actor, 'form.submission.export', form)
])
- .then(([ fields, rows, selectValues, decryptor ]) => {
+ .then(([ fields, selectValues, decryptor ]) => {
const filename = sanitize(form.xmlFormId);
const extension = (rootOnly === true) ? 'csv' : 'csv.zip';
response.append('Content-Disposition', contentDisposition(`${filename}.${extension}`));
response.append('Content-Type', (rootOnly === true) ? 'text/csv' : 'application/zip');
- const envelope = (rootOnly === true) ? identity : zipStreamFromParts;
- return envelope(streamBriefcaseCsvs(rows, fields, form.xmlFormId, selectValues, decryptor, rootOnly, options));
+ const envelope = (rootOnly === true) ? call : zipStreamFromParts;
+ return envelope(() => Submissions.streamForExport(form.id, draft, Object.keys(passphrases), options)
+ .then((rows) => streamBriefcaseCsvs(rows, fields, form.xmlFormId, selectValues, decryptor, rootOnly, options)));
});
});
diff --git a/lib/util/zip.js b/lib/util/zip.js
index 5fcf96933..1c6823988 100644
--- a/lib/util/zip.js
+++ b/lib/util/zip.js
@@ -14,6 +14,7 @@
const { Readable } = require('stream');
const { PartialPipe } = require('./stream');
const archiver = require('archiver');
+const { resolve } = require('./promise');
// Returns an object that can add files to an archive, without having that archive
// object directly nor knowing what else is going into it. Call append() to add a
@@ -36,8 +37,7 @@ const zipPart = () => {
// if the final component in the pipeline emitted the error, archiver would then
// emit it again, but if it was an intermediate component archiver wouldn't know
// about it. by manually aborting, we always emit the error and archiver never does.
-const zipStreamFromParts = (...zipParts) => {
- let completed = 0;
+const zipStreamFromParts = (...zipPartFns) => {
const resultStream = archiver('zip', { zlib: { level: 9 } });
// track requested callbacks and call them when they are fully added to the zip.
@@ -47,29 +47,35 @@ const zipStreamFromParts = (...zipParts) => {
if (cb != null) cb();
});
- for (const part of zipParts) {
- part.stream.on('data', ({ stream, options, cb }) => {
- const s = (stream instanceof PartialPipe)
- ? stream.pipeline((err) => { resultStream.emit('error', err); resultStream.abort(); })
- : stream;
+ const next = () => {
+ if (zipPartFns.length === 0) {
+ resultStream.finalize();
+ return;
+ }
- if (cb == null) {
- resultStream.append(s, options);
- } else {
- // using the String object will result in still an empty comment, but allows
- // separate instance equality check when the entry is recorded.
- const sentinel = new String(); // eslint-disable-line no-new-wrappers
- callbacks.set(sentinel, cb);
- resultStream.append(s, Object.assign(options, { comment: sentinel }));
- }
- });
- part.stream.on('error', (err) => { resultStream.emit('error', err); });
- part.stream.on('end', () => { // eslint-disable-line no-loop-func
- completed += 1;
- if (completed === zipParts.length)
- resultStream.finalize();
- });
- }
+ resolve(zipPartFns.shift()())
+ .then((part) => {
+ part.stream.on('data', ({ stream, options, cb }) => {
+ const s = (stream instanceof PartialPipe)
+ ? stream.pipeline((err) => { resultStream.emit('error', err); resultStream.abort(); })
+ : stream;
+
+ if (cb == null) {
+ resultStream.append(s, options);
+ } else {
+ // using the String object will result in still an empty comment, but allows
+ // separate instance equality check when the entry is recorded.
+ const sentinel = new String(); // eslint-disable-line no-new-wrappers
+ callbacks.set(sentinel, cb);
+ resultStream.append(s, Object.assign(options, { comment: sentinel }));
+ }
+ });
+ part.stream.on('error', (err) => { resultStream.emit('error', err); });
+ part.stream.on('end', next);
+ })
+ .catch((err) => { resultStream.emit('error', err); });
+ };
+ next();
return resultStream;
};
diff --git a/package-lock.json b/package-lock.json
index e5965cf1d..ce1763bd0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -5985,26 +5985,26 @@
"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
},
"slonik": {
- "version": "23.6.0",
- "resolved": "https://registry.npmjs.org/slonik/-/slonik-23.6.0.tgz",
- "integrity": "sha512-4SqZ4U9NVd6OYIsMKN2wrNbmXQqiifu54M3SP33XjXcJ8qsk2NBiwGwSGIeuW5QtRqnfNHBdgdQsNfrfMFKlag==",
+ "version": "npm:@getodk/slonik@23.6.0-1",
+ "resolved": "https://registry.npmjs.org/@getodk/slonik/-/slonik-23.6.0-1.tgz",
+ "integrity": "sha512-dZwX3lQBLkXPOfZB/RQvJ41xGWRskP4VV8eX/0y2X0DcGa43BcQwWQayEJNI8PYL6V4cse6LulPnVDdDIFSRDQ==",
"requires": {
"concat-stream": "^2.0.0",
"delay": "^5.0.0",
"es6-error": "^4.1.1",
"get-stack-trace": "^2.0.3",
- "hyperid": "^2.1.0",
+ "hyperid": "2.1.0",
"is-plain-object": "^5.0.0",
"iso8601-duration": "^1.3.0",
"pg": "^8.5.1",
"pg-connection-string": "^2.4.0",
"pg-copy-streams": "^5.1.1",
- "pg-copy-streams-binary": "^2.0.1",
+ "pg-copy-streams-binary": "2.0.1",
"pg-cursor": "^2.5.2",
"postgres-array": "^3.0.1",
"postgres-interval": "^3.0.0",
- "roarr": "^4.0.11",
- "serialize-error": "^8.0.1",
+ "roarr": "4.0.11",
+ "serialize-error": "8.0.1",
"through2": "^4.0.2"
},
"dependencies": {
diff --git a/package.json b/package.json
index e2ff00157..c9ef5bc00 100644
--- a/package.json
+++ b/package.json
@@ -35,7 +35,7 @@
"prompt": "~1",
"ramda": "~0",
"sanitize-filename": "~1",
- "slonik": "~23",
+ "slonik": "npm:@getodk/slonik@23.6.0-1",
"slonik-sql-tag-raw": "1.0.3",
"tmp-promise": "~3",
"uuid": "~3",
diff --git a/test/integration/api/submissions.js b/test/integration/api/submissions.js
index 852d078d3..9d75ba995 100644
--- a/test/integration/api/submissions.js
+++ b/test/integration/api/submissions.js
@@ -5,7 +5,7 @@ const { sql } = require('slonik');
const { createReadStream, readFileSync } = require('fs');
const { testService } = require('../setup');
const testData = require('../../data/xml');
-const { zipStreamToFiles } = require('../../util/zip');
+const { pZipStreamToFiles } = require('../../util/zip');
const { Form } = require(appRoot + '/lib/model/frames');
const { exhaust } = require(appRoot + '/lib/worker/worker');
@@ -1186,12 +1186,11 @@ describe('api: /forms/:id/submissions', () => {
}))));
it('should return the csv header even if there is no data', testService((service) =>
- service.login('alice', (asAlice) => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => {
+ service.login('alice', (asAlice) => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n');
- done();
- })))));
+ }))));
it('should return a zipfile with the relevant data', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1207,12 +1206,11 @@ describe('api: /forms/:id/submissions', () => {
.send(testData.instances.simple.three)
.set('Content-Type', 'text/xml')
.expect(200))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
result['simple.csv'].should.be.a.SimpleCsv();
- done();
- }))))));
+ })))));
it('should include all repeat rows @slow', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1236,13 +1234,12 @@ describe('api: /forms/:id/submissions', () => {
`)
.set('Content-Type', 'text/xml')
.expect(200)))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/single-repeat-1-instance-10qs/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/single-repeat-1-instance-10qs/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.eql([ 'single-repeat-1-instance-10qs.csv', 'single-repeat-1-instance-10qs-repeat.csv' ]);
result['single-repeat-1-instance-10qs.csv'].split('\n').length.should.equal(52);
result['single-repeat-1-instance-10qs-repeat.csv'].split('\n').length.should.equal(52);
- done();
- })))))));
+ }))))));
it('should not include data from other forms', testService((service) =>
service.login('alice', (asAlice) => Promise.all([
@@ -1263,8 +1260,8 @@ describe('api: /forms/:id/submissions', () => {
.set('Content-Type', 'text/xml')
.expect(200))
])
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const csv = result['simple.csv'].split('\n').map((row) => row.split(','));
csv.length.should.equal(4); // header + 2 data rows + newline
@@ -1274,8 +1271,7 @@ describe('api: /forms/:id/submissions', () => {
csv[2].shift().should.be.an.recentIsoDate();
csv[2].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]);
csv[3].should.eql([ '' ]);
- done();
- }))))));
+ })))));
it('should return a submitter-filtered zipfile with the relevant data', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1292,15 +1288,14 @@ describe('api: /forms/:id/submissions', () => {
.send(testData.instances.simple.three)
.set('Content-Type', 'text/xml')
.expect(200))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/submitterId eq 5'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/submitterId eq 5'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const lines = result['simple.csv'].split('\n');
lines.length.should.equal(4);
lines[1].endsWith(',three,Chelsea,38,three,5,Alice,0,0,,,,0,').should.equal(true);
lines[2].endsWith(',one,Alice,30,one,5,Alice,0,0,,,,0,').should.equal(true);
- done();
- })))))));
+ }))))));
it('should return a review state-filtered zipfile with the relevant data', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1319,15 +1314,14 @@ describe('api: /forms/:id/submissions', () => {
.send(testData.instances.simple.three)
.set('Content-Type', 'text/xml')
.expect(200))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/reviewState eq null'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/reviewState eq null'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const lines = result['simple.csv'].split('\n');
lines.length.should.equal(4);
lines[1].endsWith(',three,Chelsea,38,three,5,Alice,0,0,,,,0,').should.equal(true);
lines[2].endsWith(',one,Alice,30,one,5,Alice,0,0,,,,0,').should.equal(true);
- done();
- }))))));
+ })))));
it('should return a submissionDate-filtered zipfile with the relevant data', testService((service, { run }) =>
service.login('alice', (asAlice) =>
@@ -1341,14 +1335,13 @@ describe('api: /forms/:id/submissions', () => {
.send(testData.instances.simple.two)
.set('Content-Type', 'text/xml')
.expect(200))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=year(__system/submissionDate) eq 2010'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=year(__system/submissionDate) eq 2010'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const lines = result['simple.csv'].split('\n');
lines.length.should.equal(3);
lines[1].endsWith(',one,Alice,30,one,5,Alice,0,0,,,,0,').should.equal(true);
- done();
- })))))));
+ }))))));
it('should return an updatedAt-filtered zipfile with the relevant data', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1363,14 +1356,13 @@ describe('api: /forms/:id/submissions', () => {
.then(() => asAlice.patch('/v1/projects/1/forms/simple/submissions/two')
.send({ reviewState: 'approved' })
.expect(200))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/updatedAt eq null'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/updatedAt eq null'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const lines = result['simple.csv'].split('\n');
lines.length.should.equal(3);
lines[1].endsWith(',one,Alice,30,one,5,Alice,0,0,,,,0,').should.equal(true);
- done();
- }))))));
+ })))));
it('should return a zipfile with the relevant attachments', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1388,8 +1380,8 @@ describe('api: /forms/:id/submissions', () => {
.attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' })
.attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' })
.expect(201))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([
'binaryType.csv',
'media/my_file1.mp4',
@@ -1404,9 +1396,7 @@ describe('api: /forms/:id/submissions', () => {
csv[0].should.equal('SubmissionDate,meta-instanceID,file1,file2,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion');
csv[1].should.endWith(',both,my_file1.mp4,here_is_file2.jpg,both,5,Alice,2,2,,,,0,');
csv.length.should.equal(3); // newline at end
-
- done();
- })))))));
+ }))))));
it('should filter attachments by the query', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1425,14 +1415,13 @@ describe('api: /forms/:id/submissions', () => {
.attach('xml_submission_file', Buffer.from(testData.instances.binaryType.two), { filename: 'data.xml' })
.attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' })
.expect(201))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip?$filter=__system/submitterId eq 5'), (result) => {
- result.filenames.should.eql([
- 'binaryType.csv',
- 'media/my_file1.mp4'
- ]);
- done();
- }))))))));
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip?$filter=__system/submitterId eq 5'))
+ .then((result) => {
+ result.filenames.should.eql([
+ 'binaryType.csv',
+ 'media/my_file1.mp4'
+ ]);
+ })))))));
it('should list the original submitted form version per submission', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1461,15 +1450,14 @@ describe('api: /forms/:id/submissions', () => {
.send(testData.instances.simple.three.replace('id="simple"', 'id="simple" version="updated"'))
.set('Content-Type', 'text/xml')
.expect(200))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const lines = result['simple.csv'].split('\n');
lines[1].endsWith('0,updated').should.equal(true);
lines[2].endsWith('0,').should.equal(true);
lines[3].endsWith('1,').should.equal(true);
- done();
- }))))));
+ })))));
it('should split select multiple values if ?splitSelectMultiples=true', testService((service, container) =>
service.login('alice', (asAlice) =>
@@ -1483,8 +1471,8 @@ describe('api: /forms/:id/submissions', () => {
.set('Content-Type', 'application/xml')
.send(testData.instances.selectMultiple.two))
.then(() => exhaust(container))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true'))
+ .then((result) => {
result.filenames.should.containDeep([ 'selectMultiple.csv' ]);
const lines = result['selectMultiple.csv'].split('\n');
lines[0].should.equal('SubmissionDate,q1,q1/a,q1/b,g1-q2,g1-q2/m,g1-q2/x,g1-q2/y,g1-q2/z,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion');
@@ -1492,8 +1480,7 @@ describe('api: /forms/:id/submissions', () => {
.should.equal(',b,0,1,m x,1,1,0,0,two,5,Alice,0,0,,,,0,');
lines[2].slice('yyyy-mm-ddThh:mm:ss._msZ'.length)
.should.equal(',a b,1,1,x y z,0,1,1,1,one,5,Alice,0,0,,,,0,');
- done();
- }))))));
+ })))));
it('should omit multiples it does not know about', testService((service, container) =>
service.login('alice', (asAlice) =>
@@ -1506,8 +1493,8 @@ describe('api: /forms/:id/submissions', () => {
.then(() => asAlice.post('/v1/projects/1/forms/selectMultiple/submissions')
.set('Content-Type', 'application/xml')
.send(testData.instances.selectMultiple.two))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true'))
+ .then((result) => {
result.filenames.should.containDeep([ 'selectMultiple.csv' ]);
const lines = result['selectMultiple.csv'].split('\n');
lines[0].should.equal('SubmissionDate,q1,g1-q2,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion');
@@ -1515,8 +1502,7 @@ describe('api: /forms/:id/submissions', () => {
.should.equal(',b,m x,two,5,Alice,0,0,,,,0,');
lines[2].slice('yyyy-mm-ddThh:mm:ss._msZ'.length)
.should.equal(',a b,x y z,one,5,Alice,0,0,,,,0,');
- done();
- }))))));
+ })))));
it('should split select multiples and filter given both options', testService((service, container) =>
service.login('alice', (asAlice) =>
@@ -1532,15 +1518,14 @@ describe('api: /forms/:id/submissions', () => {
.then(() => asAlice.patch('/v1/projects/1/forms/selectMultiple/submissions/two')
.send({ reviewState: 'approved' }))
.then(() => exhaust(container))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true&$filter=__system/reviewState eq null'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true&$filter=__system/reviewState eq null'))
+ .then((result) => {
const lines = result['selectMultiple.csv'].split('\n');
lines.length.should.equal(3);
lines[1].should.containEql(',one,');
lines[1].should.not.containEql('two');
lines[2].should.equal('');
- done();
- }))))));
+ })))));
it('should export deleted fields and values if ?deletedFields=true', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1573,8 +1558,8 @@ describe('api: /forms/:id/submissions', () => {
.then(() => asAlice.post('/v1/projects/1/forms/simple/submissions')
.set('Content-Type', 'application/xml')
.send(testData.instances.simple.three.replace('id="simple"', 'id="simple" version="2"')))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?deletedFields=true'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?deletedFields=true'))
+ .then((result) => {
result.filenames.should.containDeep([ 'simple.csv' ]);
const lines = result['simple.csv'].split('\n');
lines[0].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion');
@@ -1584,8 +1569,7 @@ describe('api: /forms/:id/submissions', () => {
.should.equal(',two,Bob,34,two,5,Alice,0,0,,,,0,');
lines[3].slice('yyyy-mm-ddThh:mm:ss._msZ'.length)
.should.equal(',one,Alice,30,one,5,Alice,0,0,,,,0,');
- done();
- }))))));
+ })))));
it('should skip attachments if ?attachments=false is given', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1603,15 +1587,14 @@ describe('api: /forms/:id/submissions', () => {
.attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' })
.attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' })
.expect(201))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip?attachments=false'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip?attachments=false'))
+ .then((result) => {
result.filenames.should.containDeep([ 'binaryType.csv' ]);
should.not.exist(result['media/my_file1.mp4']);
should.not.exist(result['media/here_is_file2.jpg']);
- done();
- })))))));
+ }))))));
it('should give the appropriate filename if ?attachments=false is given', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1636,12 +1619,11 @@ describe('api: /forms/:id/submissions', () => {
.set('X-OpenRosa-Version', '1.0')
.attach('xml_submission_file', Buffer.from(testData.instances.simple.one), { filename: 'data.xml' })
.expect(201)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?groupPaths=false'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?groupPaths=false'))
+ .then((result) => {
const csv = result['simple.csv'].split('\n');
csv[0].should.equal('SubmissionDate,instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion');
- done();
- }))))));
+ })))));
it('should split select AND omit group paths given both options', testService((service, container) =>
service.login('alice', (asAlice) =>
@@ -1655,8 +1637,8 @@ describe('api: /forms/:id/submissions', () => {
.set('Content-Type', 'application/xml')
.send(testData.instances.selectMultiple.two))
.then(() => exhaust(container))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true&groupPaths=false'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true&groupPaths=false'))
+ .then((result) => {
result.filenames.should.containDeep([ 'selectMultiple.csv' ]);
const lines = result['selectMultiple.csv'].split('\n');
lines[0].should.equal('SubmissionDate,q1,q1/a,q1/b,q2,q2/m,q2/x,q2/y,q2/z,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion');
@@ -1664,8 +1646,7 @@ describe('api: /forms/:id/submissions', () => {
.should.equal(',b,0,1,m x,1,1,0,0,two,5,Alice,0,0,,,,0,');
lines[2].slice('yyyy-mm-ddThh:mm:ss._msZ'.length)
.should.equal(',a b,1,1,x y z,0,1,1,1,one,5,Alice,0,0,,,,0,');
- done();
- }))))));
+ })))));
it('should properly count present attachments', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1678,8 +1659,8 @@ describe('api: /forms/:id/submissions', () => {
.attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' })
.attach('my_file1.mp4', Buffer.from('this is test file one'), { filename: 'my_file1.mp4' })
.expect(201)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([
'binaryType.csv',
'media/my_file1.mp4'
@@ -1691,8 +1672,7 @@ describe('api: /forms/:id/submissions', () => {
csv[1].should.endWith(',both,my_file1.mp4,here_is_file2.jpg,both,5,Alice,1,2,,,,0,');
csv.length.should.equal(3); // newline at end
- done();
- })))))));
+ }))))));
it('should return worker-processed consolidated client audit log attachments', testService((service, container) =>
service.login('alice', (asAlice) =>
@@ -1711,8 +1691,8 @@ describe('api: /forms/:id/submissions', () => {
.attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' })
.expect(201))
.then(() => exhaust(container))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -1730,9 +1710,7 @@ two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);
-
- done();
- })))
+ }))
.then(() => container.oneFirst(sql`select count(*) from client_audits`)
.then((count) => { count.should.equal(8); })))));
@@ -1752,8 +1730,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' })
.attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' })
.expect(201))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -1771,9 +1749,7 @@ two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);
-
- done();
- }))))));
+ })))));
it('should return consolidated client audit log filtered by user', testService((service, container) =>
service.login('alice', (asAlice) =>
@@ -1792,8 +1768,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' })
.attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' })
.expect(201))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip?$filter=__system/submitterId eq 5'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip?$filter=__system/submitterId eq 5'))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -1808,8 +1784,7 @@ one,d,/data/d,2000-01-01T00:10,,10,11,12,gg,
one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
`);
- done();
- })))))));
+ }))))));
it('should return consolidated client audit log filtered by review state', testService((service, container) =>
service.login('alice', (asAlice) =>
@@ -1830,8 +1805,8 @@ one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
.attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' })
.attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' })
.expect(201))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip?$filter=__system/reviewState eq \'approved\''), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip?$filter=__system/reviewState eq \'approved\''))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -1846,8 +1821,7 @@ one,d,/data/d,2000-01-01T00:10,,10,11,12,gg,
one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
`);
- done();
- }))))));
+ })))));
it('should return the latest attached audit log after openrosa replace', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1867,8 +1841,8 @@ one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
.expect(201))
.then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -1881,8 +1855,7 @@ one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);
- done();
- })))))));
+ }))))));
it('should return the latest attached audit log after REST replace', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1901,8 +1874,8 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.expect(200))
.then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -1915,8 +1888,7 @@ one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);
- done();
- })))))));
+ }))))));
it('should tolerate differences in line lengths', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1931,8 +1903,8 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.expect(201))
.then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -1946,9 +1918,7 @@ one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff
one,d,/data/d,2000-01-01T00:10,,10,11,12,,
one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
`);
-
- done();
- })))))));
+ }))))));
it('should tolerate quote inside unquoted field of client audit log', testService((service) =>
service.login('alice', (asAlice) =>
@@ -1963,8 +1933,8 @@ one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
.expect(201))
.then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -1978,9 +1948,7 @@ one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff
one,d,/data/d,2000-01-01T00:10,,10,11,12,"g""g",
one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
`);
-
- done();
- })))))));
+ }))))));
context('versioning', () => {
const withClientAuditIds = (deprecatedId, instanceId) => testData.instances.clientAudits.one
@@ -2004,8 +1972,8 @@ one,e,/data/e,2000-01-01T00:11,,,,,hh,ii
.expect(201))
.then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -2017,9 +1985,7 @@ one,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);
-
- done();
- })))))));
+ }))))));
});
it('should log the action in the audit log', testService((service) =>
@@ -2166,8 +2132,8 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.expect(200))
.then(() => asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([ 'simple.csv' ]);
const csv = result['simple.csv'].split('\n').map((row) => row.split(','));
@@ -2175,9 +2141,7 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
csv[0].should.eql([ 'SubmissionDate', 'meta-instanceID', 'name', 'age', 'KEY', 'SubmitterID', 'SubmitterName', 'AttachmentsPresent', 'AttachmentsExpected', 'Status', 'ReviewState', 'DeviceID', 'Edits', 'FormVersion' ]);
csv[1].shift().should.be.an.recentIsoDate();
csv[1].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]);
-
- done();
- })))))));
+ }))))));
it('should not include draft submissions in nondraft csvzip', testService((service) =>
service.login('alice', (asAlice) =>
@@ -2189,13 +2153,12 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.expect(200))
.then(() => asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([ 'simple.csv' ]);
result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n');
- done();
- })))))));
+ }))))));
it('should not carry draft submissions forward to the published version upon publish', testService((service) =>
service.login('alice', (asAlice) =>
@@ -2209,13 +2172,12 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.expect(200))
.then(() => asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([ 'simple.csv' ]);
result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n');
- done();
- })))))));
+ }))))));
it('should not carry over drafts when a draft is replaced', testService((service) =>
service.login('alice', (asAlice) =>
@@ -2229,13 +2191,12 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.expect(200))
.then(() => asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([ 'simple.csv' ]);
result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n');
- done();
- })))))));
+ }))))));
it('should not resurface drafts when a draft is recreated', testService((service) =>
service.login('alice', (asAlice) =>
@@ -2251,13 +2212,12 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.expect(200))
.then(() => asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip')
.expect(200)
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'))
+ .then((result) => {
result.filenames.should.containDeep([ 'simple.csv' ]);
result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n');
- done();
- })))))));
+ }))))));
it('should not log the action in the audit log', testService((service) =>
service.login('alice', (asAlice) =>
@@ -2288,8 +2248,8 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.set('X-OpenRosa-Version', '1.0')
.attach('xml_submission_file', Buffer.from(testData.instances.selectMultiple.two), { filename: 'data.xml' }))))
.then(() => exhaust(container))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/draft/submissions.csv.zip?splitSelectMultiples=true'), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/draft/submissions.csv.zip?splitSelectMultiples=true'))
+ .then((result) => {
result.filenames.should.containDeep([ 'selectMultiple.csv' ]);
const lines = result['selectMultiple.csv'].split('\n');
lines[0].should.equal('SubmissionDate,q1,q1/a,q1/b,g1-q2,g1-q2/m,g1-q2/x,g1-q2/y,g1-q2/z,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion');
@@ -2297,8 +2257,7 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.should.equal(',b,0,1,m x,1,1,0,0,two,,,0,0,,,,0,');
lines[2].slice('yyyy-mm-ddThh:mm:ss._msZ'.length)
.should.equal(',a b,1,1,x y z,0,1,1,1,one,,,0,0,,,,0,');
- done();
- }))))));
+ })))));
});
describe('GET', () => {
diff --git a/test/integration/other/encryption.js b/test/integration/other/encryption.js
index d134548db..1f03228a5 100644
--- a/test/integration/other/encryption.js
+++ b/test/integration/other/encryption.js
@@ -5,7 +5,7 @@ const { sql } = require('slonik');
const { toText } = require('streamtest').v2;
const { testService, testContainerFullTrx, testContainer } = require(appRoot + '/test/integration/setup');
const testData = require(appRoot + '/test/data/xml');
-const { zipStreamToFiles } = require(appRoot + '/test/util/zip');
+const { pZipStreamToFiles } = require(appRoot + '/test/util/zip');
const { Form, Key, Submission } = require(appRoot + '/lib/model/frames');
const { mapSequential } = require(appRoot + '/test/util/util');
const { exhaust } = require(appRoot + '/lib/worker/worker');
@@ -212,12 +212,11 @@ describe('managed encryption', () => {
.then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys')
.expect(200)
.then(({ body }) => body[0].id))
- .then((keyId) => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => {
+ .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
result['simple.csv'].should.be.an.EncryptedSimpleCsv();
- done();
- }))))));
+ })))));
it('should decrypt to CSV successfully as a direct root table', testService((service) =>
service.login('alice', (asAlice) =>
@@ -251,14 +250,13 @@ describe('managed encryption', () => {
.then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys')
.expect(200)
.then(({ body }) => body[0].id))
- .then((keyId) => new Promise((done) =>
- zipStreamToFiles(asAlice.post(`/v1/projects/1/forms/simple/submissions.csv.zip`)
+ .then((keyId) => pZipStreamToFiles(asAlice.post(`/v1/projects/1/forms/simple/submissions.csv.zip`)
.send(`${keyId}=supersecret`)
- .set('Content-Type', 'application/x-www-form-urlencoded'), (result) => {
+ .set('Content-Type', 'application/x-www-form-urlencoded'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
result['simple.csv'].should.be.an.EncryptedSimpleCsv();
- done();
- }))))));
+ })))));
it('should decrypt over cookie auth with passphrases provided via url-encoded POST body', testService((service) =>
service.login('alice', (asAlice) =>
@@ -280,16 +278,15 @@ describe('managed encryption', () => {
.expect(200)
.then(({ body }) => body)
]))
- .then(([ keyId, session ]) => new Promise((done) =>
- zipStreamToFiles(service.post(`/v1/projects/1/forms/simple/submissions.csv.zip`)
+ .then(([ keyId, session ]) => pZipStreamToFiles(service.post(`/v1/projects/1/forms/simple/submissions.csv.zip`)
.send(`${keyId}=supersecret&__csrf=${session.csrf}`)
.set('Cookie', `__Host-session=${session.token}`)
.set('X-Forwarded-Proto', 'https')
- .set('Content-Type', 'application/x-www-form-urlencoded'), (result) => {
+ .set('Content-Type', 'application/x-www-form-urlencoded'))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
result['simple.csv'].should.be.an.EncryptedSimpleCsv();
- done();
- }))))));
+ })))));
it('should decrypt with passphrases provide via JSON POST body', testService((service) =>
service.login('alice', (asAlice) =>
@@ -305,13 +302,12 @@ describe('managed encryption', () => {
.then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys')
.expect(200)
.then(({ body }) => body[0].id))
- .then((keyId) => new Promise((done) =>
- zipStreamToFiles(asAlice.post(`/v1/projects/1/forms/simple/submissions.csv.zip`)
- .send({ [keyId]: 'supersecret' }), (result) => {
+ .then((keyId) => pZipStreamToFiles(asAlice.post(`/v1/projects/1/forms/simple/submissions.csv.zip`)
+ .send({ [keyId]: 'supersecret' }))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
result['simple.csv'].should.be.an.EncryptedSimpleCsv();
- done();
- }))))));
+ })))));
it('should decrypt attached files successfully', testService((service) =>
service.login('alice', (asAlice) =>
@@ -326,16 +322,15 @@ describe('managed encryption', () => {
.then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys')
.expect(200)
.then(({ body }) => body[0].id))
- .then((keyId) => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => {
+ .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`))
+ .then((result) => {
result.filenames.length.should.equal(4);
result.filenames.should.containDeep([ 'simple.csv', 'media/alpha', 'media/beta', 'media/charlie' ]);
result['media/alpha'].should.equal('hello this is file alpha');
result['media/beta'].should.equal('and beta');
result['media/charlie'].should.equal('file charlie is right here');
- done();
- }))))));
+ })))));
it('should strip .enc suffix from decrypted attachments', testService((service) =>
service.login('alice', (asAlice) =>
@@ -349,14 +344,13 @@ describe('managed encryption', () => {
.then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys')
.expect(200)
.then(({ body }) => body[0].id))
- .then((keyId) => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => {
+ .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`))
+ .then((result) => {
result.filenames.length.should.equal(2);
result.filenames.should.containDeep([ 'simple.csv', 'media/testfile.jpg' ]);
result['media/testfile.jpg'].should.equal('hello this is a suffixed file');
- done();
- })))))));
+ }))))));
it('should decrypt client audit log attachments', testService((service, container) =>
service.login('alice', (asAlice) =>
@@ -396,8 +390,8 @@ describe('managed encryption', () => {
.then(() => asAlice.get('/v1/projects/1/forms/audits/submissions/keys')
.expect(200)
.then(({ body }) => body[0].id))
- .then((keyId) => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/audits/submissions.csv.zip?${keyId}=supersecret`), (result) => {
+ .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/audits/submissions.csv.zip?${keyId}=supersecret`))
+ .then((result) => {
result.filenames.should.containDeep([
'audits.csv',
'media/audit.csv',
@@ -415,9 +409,7 @@ two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb
two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd
two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
`);
-
- done();
- }))))));
+ })))));
it('should handle mixed [plaintext/encrypted] attachments (not decrypting)', testService((service) =>
service.login('alice', (asAlice) =>
@@ -439,14 +431,13 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.expect(200)
.then(({ text }) => sendEncrypted(asAlice, extractVersion(text), extractPubkey(text)))
.then((send) => send(testData.instances.binaryType.two, { 'here_is_file2.jpg': 'file two you cant see' })))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/binaryType/submissions.csv.zip`), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/binaryType/submissions.csv.zip`))
+ .then((result) => {
result.filenames.length.should.equal(2);
result.filenames.should.containDeep([ 'binaryType.csv', 'media/my_file1.mp4' ]);
result['media/my_file1.mp4'].should.equal('this is file one');
- done();
- }))))));
+ })))));
it('should handle mixed [plaintext/encrypted] attachments (decrypting)', testService((service) =>
service.login('alice', (asAlice) =>
@@ -471,15 +462,14 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.then(() => asAlice.get('/v1/projects/1/forms/binaryType/submissions/keys')
.expect(200)
.then(({ body }) => body[0].id))
- .then((keyId) => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/binaryType/submissions.csv.zip?${keyId}=supersecret`), (result) => {
+ .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/binaryType/submissions.csv.zip?${keyId}=supersecret`))
+ .then((result) => {
result.filenames.length.should.equal(3);
result.filenames.should.containDeep([ 'binaryType.csv', 'media/my_file1.mp4', 'media/here_is_file2.jpg' ]);
result['media/my_file1.mp4'].should.equal('this is file one');
result['media/here_is_file2.jpg'].should.equal('file two you can see');
- done();
- }))))));
+ })))));
it('should handle mixed[plaintext/encrypted] formdata (decrypting)', testService((service) =>
service.login('alice', (asAlice) =>
@@ -498,8 +488,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys')
.expect(200)
.then(({ body }) => body[0].id))
- .then((keyId) => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => {
+ .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const csv = result['simple.csv'].split('\n').map((row) => row.split(','));
csv.length.should.equal(5); // header + 3 data rows + newline
@@ -513,8 +503,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
csv[3].shift().should.be.an.recentIsoDate();
csv[3].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]);
csv[4].should.eql([ '' ]);
- done();
- })))))));
+ }))))));
it('should handle mixed[plaintext/encrypted] formdata (not decrypting)', testService((service, { Project, FormPartial }) =>
service.login('alice', (asAlice) =>
@@ -530,8 +519,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.then(({ text }) => sendEncrypted(asAlice, extractVersion(text), extractPubkey(text)))
.then((send) => send(testData.instances.simple.two)
.then(() => send(testData.instances.simple.three))))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip`), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip`))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const csv = result['simple.csv'].split('\n').map((row) => row.split(','));
@@ -546,8 +535,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
csv[3].shift().should.be.an.recentIsoDate();
csv[3].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]);
csv[4].should.eql([ '' ]);
- done();
- })))))));
+ }))))));
// we have to sort of cheat at this to get two different managed keys in effect.
it('should handle mixed[managedA/managedB] formdata (decrypting)', testService((service, { Forms, Projects }) =>
@@ -585,8 +573,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys')
.expect(200)
.then(({ body }) => body.map((key) => key.id)))
- .then((keyIds) => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyIds[1]}=supersecret&${keyIds[0]}=superdupersecret`), (result) => {
+ .then((keyIds) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyIds[1]}=supersecret&${keyIds[0]}=superdupersecret`))
+ .then((result) => {
const csv = result['simple.csv'].split('\n').map((row) => row.split(','));
csv.length.should.equal(5); // header + 3 data rows + newline
csv[0].should.eql([ 'SubmissionDate', 'meta-instanceID', 'name', 'age', 'KEY', 'SubmitterID', 'SubmitterName', 'AttachmentsPresent', 'AttachmentsExpected', 'Status', 'ReviewState', 'DeviceID', 'Edits', 'FormVersion' ]);
@@ -600,8 +588,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
csv[3].pop().should.match(/^\[encrypted:........\]$/);
csv[3].should.eql([ 'one','Alice','30','one','5','Alice','1','1','','','','0' ]);
csv[4].should.eql([ '' ]);
- done();
- }))))));
+ })))));
it('should handle mixed [plaintext/missing-encrypted-xml] formdata (decrypting)', testService((service, { Project, FormPartial }) =>
service.login('alice', (asAlice) =>
@@ -622,8 +609,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys')
.expect(200)
.then(({ body }) => body[0].id))
- .then((keyId) => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => {
+ .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const csv = result['simple.csv'].split('\n').map((row) => row.split(','));
@@ -635,8 +622,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
csv[2].shift().should.be.an.recentIsoDate();
csv[2].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]);
csv[3].should.eql([ '' ]);
- done();
- })))))));
+ }))))));
it('should handle mixed [plaintext/missing-encrypted-xml] formdata (not decrypting)', testService((service, { Project, FormPartial }) =>
service.login('alice', (asAlice) =>
@@ -654,8 +640,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
.send(envelope)
.set('Content-Type', 'text/xml')
.expect(200)))
- .then(() => new Promise((done) =>
- zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip`), (result) => {
+ .then(() => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip`))
+ .then((result) => {
result.filenames.should.eql([ 'simple.csv' ]);
const csv = result['simple.csv'].split('\n').map((row) => row.split(','));
@@ -667,8 +653,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff
csv[2].shift().should.be.an.recentIsoDate();
csv[2].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]);
csv[3].should.eql([ '' ]);
- done();
- })))))));
+ }))))));
});
});
diff --git a/test/unit/data/attachments.js b/test/unit/data/attachments.js
index d0b7297cd..c46dee12e 100644
--- a/test/unit/data/attachments.js
+++ b/test/unit/data/attachments.js
@@ -12,7 +12,9 @@ describe('.zip attachments streaming', () => {
{ row: { instanceId: 'subone', name: 'secondfile.ext', content: 'this is my second file' } },
{ row: { instanceId: 'subtwo', name: 'thirdfile.ext', content: 'this is my third file' } }
]);
- zipStreamToFiles(zipStreamFromParts(streamAttachments(inStream)), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => streamAttachments(inStream)), (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([
'media/firstfile.ext',
'media/secondfile.ext',
@@ -33,7 +35,9 @@ describe('.zip attachments streaming', () => {
{ row: { instanceId: 'subone', name: '../secondfile.ext', content: 'this is my second file' } },
{ row: { instanceId: 'subone', name: './.secondfile.ext', content: 'this is my duplicate second file' } },
]);
- zipStreamToFiles(zipStreamFromParts(streamAttachments(inStream)), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => streamAttachments(inStream)), (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([
'media/firstfile.ext',
'media/..secondfile.ext',
@@ -48,7 +52,9 @@ describe('.zip attachments streaming', () => {
const inStream = streamTest.fromObjects([
{ row: { instanceId: 'subone', name: 'firstfile.ext.enc', content: 'this is my first file' } }
]);
- zipStreamToFiles(zipStreamFromParts(streamAttachments(inStream)), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => streamAttachments(inStream)), (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'media/firstfile.ext.enc' ]);
done();
});
@@ -58,7 +64,9 @@ describe('.zip attachments streaming', () => {
const inStream = streamTest.fromObjects([
{ row: { instanceId: 'subone', name: 'firstfile.ext.enc', content: 'this is my first file' } }
]);
- zipStreamToFiles(zipStreamFromParts(streamAttachments(inStream, () => {})), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => streamAttachments(inStream, () => {})), (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'media/firstfile.ext' ]);
done();
});
diff --git a/test/unit/data/briefcase.js b/test/unit/data/briefcase.js
index 5c43939c6..8da16ce27 100644
--- a/test/unit/data/briefcase.js
+++ b/test/unit/data/briefcase.js
@@ -28,7 +28,7 @@ const withAttachments = (present, expected, row) => ({ ...row, aux: { ...row.aux
const callAndParse = (inStream, formXml, xmlFormId, callback) => {
fieldsFor(formXml).then((fields) => {
- zipStreamToFiles(zipStreamFromParts(streamBriefcaseCsvs(inStream, fields, xmlFormId)), callback);
+ zipStreamToFiles(zipStreamFromParts(() => streamBriefcaseCsvs(inStream, fields, xmlFormId)), callback);
});
};
@@ -60,7 +60,9 @@ describe('.csv.zip briefcase output @slow', () => {
instance('three', 'Chelsea38San Francisco, CA')
]);
- callAndParse(inStream, formXml, 'mytestform', (result) => {
+ callAndParse(inStream, formXml, 'mytestform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'mytestform.csv' ]);
result['mytestform.csv'].should.equal(
`SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -82,7 +84,7 @@ describe('.csv.zip briefcase output @slow', () => {
}]);
// not hanging is the assertion here:
- callAndParse(inStream, testData.forms.simple, 'simple', () => { done(); });
+ callAndParse(inStream, testData.forms.simple, 'simple', (err) => { done(err); });
});
it('should attach submitter information if present', (done) => {
@@ -111,7 +113,9 @@ describe('.csv.zip briefcase output @slow', () => {
withSubmitter(15, 'lito', instance('three', 'Chelsea38San Francisco, CA'))
]);
- callAndParse(inStream, formXml, 'mytestform', (result) => {
+ callAndParse(inStream, formXml, 'mytestform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'mytestform.csv' ]);
result['mytestform.csv'].should.equal(
`SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -149,7 +153,9 @@ describe('.csv.zip briefcase output @slow', () => {
withAttachments(3, 3, instance('three', 'Chelsea38San Francisco, CA'))
]);
- callAndParse(inStream, formXml, 'mytestform', (result) => {
+ callAndParse(inStream, formXml, 'mytestform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'mytestform.csv' ]);
result['mytestform.csv'].should.equal(
`SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -186,7 +192,9 @@ describe('.csv.zip briefcase output @slow', () => {
{ ...instance('two', 'Bob34Portland, OR'), reviewState: 'rejected' }
]);
- callAndParse(inStream, formXml, 'mytestform', (result) => {
+ callAndParse(inStream, formXml, 'mytestform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'mytestform.csv' ]);
result['mytestform.csv'].should.equal(
`SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -221,7 +229,9 @@ describe('.csv.zip briefcase output @slow', () => {
{ ...instance('one', 'missing'), xml: 'missing', deviceId: 'test device' }
]);
- callAndParse(inStream, formXml, 'mytestform', (result) => {
+ callAndParse(inStream, formXml, 'mytestform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'mytestform.csv' ]);
result['mytestform.csv'].should.equal(
`SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -255,7 +265,9 @@ describe('.csv.zip briefcase output @slow', () => {
data.aux.edit.count = 3;
const inStream = streamTest.fromObjects([ data ]);
- callAndParse(inStream, formXml, 'mytestform', (result) => {
+ callAndParse(inStream, formXml, 'mytestform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'mytestform.csv' ]);
result['mytestform.csv'].should.equal(
`SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -291,7 +303,9 @@ describe('.csv.zip briefcase output @slow', () => {
two.aux.exports.formVersion = 'updated';
const inStream = streamTest.fromObjects([ one, two ]);
- callAndParse(inStream, formXml, 'mytestform', (result) => {
+ callAndParse(inStream, formXml, 'mytestform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'mytestform.csv' ]);
result['mytestform.csv'].should.equal(
`SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -326,7 +340,9 @@ describe('.csv.zip briefcase output @slow', () => {
instance('one', '«Alice»30Seattle, WA'),
]);
- callAndParse(inStream, formXml, 'mytestform', (result) => {
+ callAndParse(inStream, formXml, 'mytestform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'mytestform.csv' ]);
result['mytestform.csv'].should.equal(
`SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -362,7 +378,9 @@ describe('.csv.zip briefcase output @slow', () => {
instance('three', 'Chelsea38')
]);
- callAndParse(inStream, formXml, 'mytestform', (result) => {
+ callAndParse(inStream, formXml, 'mytestform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'mytestform.csv' ]);
result['mytestform.csv'].should.equal(
`SubmissionDate,name,age,location-Latitude,location-Longitude,location-Altitude,location-Accuracy,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -381,7 +399,9 @@ describe('.csv.zip briefcase output @slow', () => {
]);
fieldsFor(testData.forms.selectMultiple).then((fields) => {
- zipStreamToFiles(zipStreamFromParts(streamBriefcaseCsvs(inStream, fields, 'selectMultiple', { '/q1': [ 'x', 'y', 'z' ], '/g1/q2': [ 'm', 'n' ] })), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => streamBriefcaseCsvs(inStream, fields, 'selectMultiple', { '/q1': [ 'x', 'y', 'z' ], '/g1/q2': [ 'm', 'n' ] })), (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'selectMultiple.csv' ]);
result['selectMultiple.csv'].should.equal(
`SubmissionDate,q1,q1/x,q1/y,q1/z,g1-q2,g1-q2/m,g1-q2/n,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -429,7 +449,9 @@ describe('.csv.zip briefcase output @slow', () => {
instance('three', 'threeChelseaHouseSan Francisco, CA99 Mission Ave'),
]);
- callAndParse(inStream, formXml, 'structuredform', (result) => {
+ callAndParse(inStream, formXml, 'structuredform', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'structuredform.csv' ]);
result['structuredform.csv'].should.equal(
`SubmissionDate,meta-instanceID,name,home-type,home-address-street,home-address-city,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -478,7 +500,9 @@ describe('.csv.zip briefcase output @slow', () => {
]);
fieldsFor(formXml).then((fields) => {
- zipStreamToFiles(zipStreamFromParts(streamBriefcaseCsvs(inStream, fields, 'structuredform', undefined, undefined, false, { groupPaths: false })), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => streamBriefcaseCsvs(inStream, fields, 'structuredform', undefined, undefined, false, { groupPaths: false })), (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'structuredform.csv' ]);
result['structuredform.csv'].should.equal(
`SubmissionDate,instanceID,name,type,street,city,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -498,7 +522,9 @@ describe('.csv.zip briefcase output @slow', () => {
]);
fieldsFor(testData.forms.selectMultiple).then((fields) => {
- zipStreamToFiles(zipStreamFromParts(streamBriefcaseCsvs(inStream, fields, 'selectMultiple', { '/q1': [ 'x', 'y', 'z' ], '/g1/q2': [ 'm', 'n' ] }, undefined, false, { groupPaths: false })), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => streamBriefcaseCsvs(inStream, fields, 'selectMultiple', { '/q1': [ 'x', 'y', 'z' ], '/g1/q2': [ 'm', 'n' ] }, undefined, false, { groupPaths: false })), (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.eql([ 'selectMultiple.csv' ]);
result['selectMultiple.csv'].should.equal(
`SubmissionDate,q1,q1/x,q1/y,q1/z,q2,q2/m,q2/n,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -565,7 +591,9 @@ describe('.csv.zip briefcase output @slow', () => {
instance('three', 'threeChelsea38Candace2'),
]);
- callAndParse(inStream, formXml, 'singlerepeat', (result) => {
+ callAndParse(inStream, formXml, 'singlerepeat', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.containDeep([ 'singlerepeat.csv', 'singlerepeat-child.csv' ]);
result['singlerepeat.csv'].should.equal(
`SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -627,7 +655,9 @@ Candace,2,three,three/children/child[1]
const inStream = streamTest.fromObjects(
(new Array(127)).fill(null).map(_ => instance(uuid(), `${uuid()}${uuid()}${uuid()}`)));
- callAndParse(inStream, formXml, 'singlerepeat', (result) => {
+ callAndParse(inStream, formXml, 'singlerepeat', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.containDeep([ 'singlerepeat.csv', 'singlerepeat-child.csv' ]);
result['singlerepeat.csv'].split('\n').length.should.equal(129);
result['singlerepeat-child.csv'].split('\n').length.should.equal(129);
@@ -702,7 +732,9 @@ Candace,2,three,three/children/child[1]
instance('three', 'threeChelsea38CandaceMillennium FalconX-WingPod racer2'),
]);
- callAndParse(inStream, formXml, 'multirepeat', (result) => {
+ callAndParse(inStream, formXml, 'multirepeat', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.containDeep([ 'multirepeat.csv', 'multirepeat-child.csv', 'multirepeat-toy.csv' ]);
result['multirepeat.csv'].should.equal(
`SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -762,7 +794,9 @@ Pod racer,three/children/child[1],three/children/child[1]/toy[3]
instance('one', 'AliceBobChelseaLiving at home')
]);
- callAndParse(inStream, formXml, 'pathprefix', (result) => {
+ callAndParse(inStream, formXml, 'pathprefix', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.containDeep([ 'pathprefix.csv', 'pathprefix-children.csv' ]);
result['pathprefix.csv'].should.equal(
`SubmissionDate,name,children-status,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -851,7 +885,9 @@ Chelsea,one,one/children[2]
aux: { attachment: { present: 0, expected: 0 }, encryption: {}, edit: { count: 0 }, exports: { formVersion: '' } }
}]);
- callAndParse(inStream, formXml, 'all-data-types', (result) => {
+ callAndParse(inStream, formXml, 'all-data-types', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.containDeep([ 'all-data-types.csv' ]);
result['all-data-types.csv'].should.equal(
`SubmissionDate,some_string,some_int,some_decimal,some_date,some_time,some_date_time,some_geopoint-Latitude,some_geopoint-Longitude,some_geopoint-Altitude,some_geopoint-Accuracy,some_geotrace,some_geoshape,some_barcode,meta-instanceID,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -978,7 +1014,9 @@ Chelsea,one,one/children[2]
aux: { attachment: { present: 0, expected: 0 }, encryption: {}, edit: { count: 0 }, exports: { formVersion: '' } }
}]);
- callAndParse(inStream, formXml, 'nested-repeats', (result) => {
+ callAndParse(inStream, formXml, 'nested-repeats', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.containDeep([ 'nested-repeats.csv', 'nested-repeats-g1.csv', 'nested-repeats-g2.csv', 'nested-repeats-g3.csv' ]);
result['nested-repeats.csv'].should.equal(
`SubmissionDate,meta-instanceID,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
@@ -1064,7 +1102,9 @@ some text 3.1.4,uuid:0a1b861f-a5fd-4f49-846a-78dcf06cfc1b/g1[3]/g2[1],uuid:0a1b8
instance('three', 'threeChelseaInstantaneous FoodFerrenceMick'),
]);
- callAndParse(inStream, formXml, 'ambiguous', (result) => {
+ callAndParse(inStream, formXml, 'ambiguous', (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.containDeep([ 'ambiguous.csv', 'ambiguous-entry~1.csv', 'ambiguous-entry~2.csv' ]);
result['ambiguous.csv'].should.equal(
`SubmissionDate,meta-instanceID,name,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion
diff --git a/test/unit/util/zip.js b/test/unit/util/zip.js
index 9d723d8d7..65f056109 100644
--- a/test/unit/util/zip.js
+++ b/test/unit/util/zip.js
@@ -13,7 +13,9 @@ describe('zipPart streamer', () => {
const part = zipPart();
let closed = false;
- zipStreamToFiles(zipStreamFromParts(part), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => part), (err, result) => {
+ if(err) return done(err);
+
closed = true;
done();
});
@@ -26,27 +28,26 @@ describe('zipPart streamer', () => {
it('should close the archive successfully given no files', (done) => {
const part = zipPart();
// no assertions other than verifying that done is called.
- zipStreamToFiles(zipStreamFromParts(part), () => done());
+ zipStreamToFiles(zipStreamFromParts(() => part), (err) => done(err));
part.finalize();
});
it('should error out the archive if a part pushes an error', (done) => {
- const part1 = zipPart();
- const part2 = zipPart();
- const archive = zipStreamFromParts(part1, part2);
+ const part = zipPart();
+ const archive = zipStreamFromParts(() => part);
archive.on('error', (err) => {
err.message.should.equal('whoops');
done();
});
- part1.append('test 1', { name: 'x/test1.file' });
- part2.error(new Error('whoops'));
+ part.append('test 1', { name: 'x/test1.file' });
+ setTimeout(() => { part.error(new Error('whoops')); });
});
it('should call the given callback only when the file has been added', (done) => {
const part = zipPart();
const file = new Readable({ read() {} });
- const archive = zipStreamFromParts(part);
+ const archive = zipStreamFromParts(() => part);
let pushedAll = false;
part.append(file, { name: 'file' }, () => {
@@ -65,7 +66,7 @@ describe('zipPart streamer', () => {
const part = zipPart();
const file1 = new Readable({ read() {} });
const file2 = new Readable({ read() {} });
- const archive = zipStreamFromParts(part);
+ const archive = zipStreamFromParts(() => part);
archive.pipe(createWriteStream('/dev/null'));
archive.on('end', () => {
@@ -93,7 +94,9 @@ describe('zipPart streamer', () => {
const part1 = zipPart();
const part2 = zipPart();
- zipStreamToFiles(zipStreamFromParts(part1, part2), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => part1, () => part2), (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.containDeep([
'x/test1.file',
'x/test2.file',
@@ -122,7 +125,9 @@ describe('zipPart streamer', () => {
const part1 = zipPart();
const part2 = zipPart();
- zipStreamToFiles(zipStreamFromParts(part1, part2), (result) => {
+ zipStreamToFiles(zipStreamFromParts(() => part1, () => part2), (err, result) => {
+ if(err) return done(err);
+
result.filenames.should.containDeep([ 'test1.file', 'test2.file' ]);
result['test1.file'].should.equal('test static');
result['test2.file'].should.equal('a!test!stream!');
@@ -145,7 +150,7 @@ describe('zipPart streamer', () => {
const part1 = zipPart();
const part2 = zipPart();
- const archive = zipStreamFromParts(part1, part2);
+ const archive = zipStreamFromParts(() => part1, () => part2);
let errCount = 0;
archive.on('error', (err) => {
errCount += 1;
@@ -174,7 +179,7 @@ describe('zipPart streamer', () => {
const part1 = zipPart();
const part2 = zipPart();
- const archive = zipStreamFromParts(part1, part2);
+ const archive = zipStreamFromParts(() => part1, () => part2);
archive.on('error', (err) => {
err.message.should.equal('whoops');
done();
diff --git a/test/util/zip.js b/test/util/zip.js
index c79129fb2..1717f8f1a 100644
--- a/test/util/zip.js
+++ b/test/util/zip.js
@@ -14,31 +14,38 @@ const streamTest = require('streamtest').v2;
// …
// }
const zipStreamToFiles = (zipStream, callback) => {
- tmp.file((_, tmpfile) => {
+ tmp.file((err, tmpfile) => {
+ if(err) return callback(err);
+
const writeStream = createWriteStream(tmpfile);
zipStream.pipe(writeStream);
zipStream.on('end', () => {
setTimeout(() => {
- yauzl.open(tmpfile, { autoClose: false }, (_, zipfile) => {
+ yauzl.open(tmpfile, { autoClose: false }, (err, zipfile) => {
+ if(err) return callback(err);
+
const result = { filenames: [] };
let entries = [];
let completed = 0;
- should.exist(zipfile);
zipfile.on('entry', (entry) => entries.push(entry));
zipfile.on('end', () => {
if (entries.length === 0) {
- callback(result);
+ callback(null, result);
zipfile.close();
} else {
entries.forEach((entry) => {
result.filenames.push(entry.fileName);
- zipfile.openReadStream(entry, (_, resultStream) => {
- resultStream.pipe(streamTest.toText((_, contents) => {
+ zipfile.openReadStream(entry, (err, resultStream) => {
+ if(err) return callback(err);
+
+ resultStream.pipe(streamTest.toText((err, contents) => {
+ if(err) return callback(err);
+
result[entry.fileName] = contents;
completed += 1;
if (completed === entries.length) {
- callback(result);
+ callback(null, result);
zipfile.close();
}
}));
@@ -52,5 +59,7 @@ const zipStreamToFiles = (zipStream, callback) => {
});
};
-module.exports = { zipStreamToFiles };
+const pZipStreamToFiles = (zipStream) => new Promise((resolve, reject) => zipStreamToFiles(zipStream, (err, result) => err ? reject(err) : resolve(result)));
+
+module.exports = { zipStreamToFiles, pZipStreamToFiles };