diff --git a/lib/resources/odata.js b/lib/resources/odata.js index 1a21adadc..608907d68 100644 --- a/lib/resources/odata.js +++ b/lib/resources/odata.js @@ -64,12 +64,11 @@ module.exports = (service, endpoint) => { const options = QueryOptions.fromODataRequest(params, query); return Promise.all([ Forms.getFields(form.def.id), - Submissions.streamForExport(form.id, draft, undefined, options), ((params.table === 'Submissions') && options.hasPaging()) ? Submissions.countByFormId(form.id, draft, options) : resolve(null) ]) - .then(([ fields, stream, count ]) => - json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count))); + .then(([ fields, count ]) => Submissions.streamForExport(form.id, draft, undefined, options) + .then((stream) => json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count)))); }))); }; diff --git a/lib/resources/submissions.js b/lib/resources/submissions.js index 4232b1748..e06d55e7e 100644 --- a/lib/resources/submissions.js +++ b/lib/resources/submissions.js @@ -7,7 +7,7 @@ // including this file, may be copied, modified, propagated, or distributed // except according to the terms contained in the LICENSE file. -const { always, identity } = require('ramda'); +const { always, identity, call } = require('ramda'); const multer = require('multer'); const sanitize = require('sanitize-filename'); const { Blob, Form, Submission } = require('../model/frames'); @@ -269,20 +269,20 @@ module.exports = (service, endpoint) => { const options = QueryOptions.fromSubmissionCsvRequest(query); return Promise.all([ (options.deletedFields === true) ? Forms.getMergedFields(form.id) : Forms.getFields(form.def.id), - Submissions.streamForExport(form.id, draft, keys, options), (options.splitSelectMultiples !== true) ? null : Submissions.getSelectMultipleValuesForExport(form.id, draft, options), - SubmissionAttachments.streamForExport(form.id, draft, keys, options), - ClientAudits.streamForExport(form.id, draft, keys, options), draft ? null : Audits.log(auth.actor, 'form.submission.export', form) - ]).then(([ fields, rows, selectValues, attachments, clientAudits ]) => { + ]).then(([ fields, selectValues ]) => { const filename = sanitize(form.xmlFormId); response.append('Content-Disposition', contentDisposition(`${filename}.zip`)); response.append('Content-Type', 'application/zip'); return zipStreamFromParts( // TODO: not 100% sure that these streams close right on crash. - streamBriefcaseCsvs(rows, fields, form.xmlFormId, selectValues, decryptor, false, options), - streamAttachments(attachments, decryptor), - streamClientAudits(clientAudits, form, decryptor) + () => Submissions.streamForExport(form.id, draft, keys, options) + .then((rows) => streamBriefcaseCsvs(rows, fields, form.xmlFormId, selectValues, decryptor, false, options)), + () => SubmissionAttachments.streamForExport(form.id, draft, keys, options) + .then((attachments) => streamAttachments(attachments, decryptor)), + () => ClientAudits.streamForExport(form.id, draft, keys, options) + .then((clientAudits) => streamClientAudits(clientAudits, form, decryptor)) ); }); })); @@ -295,18 +295,18 @@ module.exports = (service, endpoint) => { const options = QueryOptions.fromSubmissionCsvRequest(query); return Promise.all([ (options.deletedFields === true) ? Forms.getMergedFields(form.id) : Forms.getFields(form.def.id), - Submissions.streamForExport(form.id, draft, Object.keys(passphrases), options), (options.splitSelectMultiples !== true) ? null : Submissions.getSelectMultipleValuesForExport(form.id, draft, options), Keys.getDecryptor(passphrases), draft ? null : Audits.log(auth.actor, 'form.submission.export', form) ]) - .then(([ fields, rows, selectValues, decryptor ]) => { + .then(([ fields, selectValues, decryptor ]) => { const filename = sanitize(form.xmlFormId); const extension = (rootOnly === true) ? 'csv' : 'csv.zip'; response.append('Content-Disposition', contentDisposition(`${filename}.${extension}`)); response.append('Content-Type', (rootOnly === true) ? 'text/csv' : 'application/zip'); - const envelope = (rootOnly === true) ? identity : zipStreamFromParts; - return envelope(streamBriefcaseCsvs(rows, fields, form.xmlFormId, selectValues, decryptor, rootOnly, options)); + const envelope = (rootOnly === true) ? call : zipStreamFromParts; + return envelope(() => Submissions.streamForExport(form.id, draft, Object.keys(passphrases), options) + .then((rows) => streamBriefcaseCsvs(rows, fields, form.xmlFormId, selectValues, decryptor, rootOnly, options))); }); }); diff --git a/lib/util/zip.js b/lib/util/zip.js index 5fcf96933..1c6823988 100644 --- a/lib/util/zip.js +++ b/lib/util/zip.js @@ -14,6 +14,7 @@ const { Readable } = require('stream'); const { PartialPipe } = require('./stream'); const archiver = require('archiver'); +const { resolve } = require('./promise'); // Returns an object that can add files to an archive, without having that archive // object directly nor knowing what else is going into it. Call append() to add a @@ -36,8 +37,7 @@ const zipPart = () => { // if the final component in the pipeline emitted the error, archiver would then // emit it again, but if it was an intermediate component archiver wouldn't know // about it. by manually aborting, we always emit the error and archiver never does. -const zipStreamFromParts = (...zipParts) => { - let completed = 0; +const zipStreamFromParts = (...zipPartFns) => { const resultStream = archiver('zip', { zlib: { level: 9 } }); // track requested callbacks and call them when they are fully added to the zip. @@ -47,29 +47,35 @@ const zipStreamFromParts = (...zipParts) => { if (cb != null) cb(); }); - for (const part of zipParts) { - part.stream.on('data', ({ stream, options, cb }) => { - const s = (stream instanceof PartialPipe) - ? stream.pipeline((err) => { resultStream.emit('error', err); resultStream.abort(); }) - : stream; + const next = () => { + if (zipPartFns.length === 0) { + resultStream.finalize(); + return; + } - if (cb == null) { - resultStream.append(s, options); - } else { - // using the String object will result in still an empty comment, but allows - // separate instance equality check when the entry is recorded. - const sentinel = new String(); // eslint-disable-line no-new-wrappers - callbacks.set(sentinel, cb); - resultStream.append(s, Object.assign(options, { comment: sentinel })); - } - }); - part.stream.on('error', (err) => { resultStream.emit('error', err); }); - part.stream.on('end', () => { // eslint-disable-line no-loop-func - completed += 1; - if (completed === zipParts.length) - resultStream.finalize(); - }); - } + resolve(zipPartFns.shift()()) + .then((part) => { + part.stream.on('data', ({ stream, options, cb }) => { + const s = (stream instanceof PartialPipe) + ? stream.pipeline((err) => { resultStream.emit('error', err); resultStream.abort(); }) + : stream; + + if (cb == null) { + resultStream.append(s, options); + } else { + // using the String object will result in still an empty comment, but allows + // separate instance equality check when the entry is recorded. + const sentinel = new String(); // eslint-disable-line no-new-wrappers + callbacks.set(sentinel, cb); + resultStream.append(s, Object.assign(options, { comment: sentinel })); + } + }); + part.stream.on('error', (err) => { resultStream.emit('error', err); }); + part.stream.on('end', next); + }) + .catch((err) => { resultStream.emit('error', err); }); + }; + next(); return resultStream; }; diff --git a/package-lock.json b/package-lock.json index e5965cf1d..ce1763bd0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5985,26 +5985,26 @@ "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, "slonik": { - "version": "23.6.0", - "resolved": "https://registry.npmjs.org/slonik/-/slonik-23.6.0.tgz", - "integrity": "sha512-4SqZ4U9NVd6OYIsMKN2wrNbmXQqiifu54M3SP33XjXcJ8qsk2NBiwGwSGIeuW5QtRqnfNHBdgdQsNfrfMFKlag==", + "version": "npm:@getodk/slonik@23.6.0-1", + "resolved": "https://registry.npmjs.org/@getodk/slonik/-/slonik-23.6.0-1.tgz", + "integrity": "sha512-dZwX3lQBLkXPOfZB/RQvJ41xGWRskP4VV8eX/0y2X0DcGa43BcQwWQayEJNI8PYL6V4cse6LulPnVDdDIFSRDQ==", "requires": { "concat-stream": "^2.0.0", "delay": "^5.0.0", "es6-error": "^4.1.1", "get-stack-trace": "^2.0.3", - "hyperid": "^2.1.0", + "hyperid": "2.1.0", "is-plain-object": "^5.0.0", "iso8601-duration": "^1.3.0", "pg": "^8.5.1", "pg-connection-string": "^2.4.0", "pg-copy-streams": "^5.1.1", - "pg-copy-streams-binary": "^2.0.1", + "pg-copy-streams-binary": "2.0.1", "pg-cursor": "^2.5.2", "postgres-array": "^3.0.1", "postgres-interval": "^3.0.0", - "roarr": "^4.0.11", - "serialize-error": "^8.0.1", + "roarr": "4.0.11", + "serialize-error": "8.0.1", "through2": "^4.0.2" }, "dependencies": { diff --git a/package.json b/package.json index e2ff00157..c9ef5bc00 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "prompt": "~1", "ramda": "~0", "sanitize-filename": "~1", - "slonik": "~23", + "slonik": "npm:@getodk/slonik@23.6.0-1", "slonik-sql-tag-raw": "1.0.3", "tmp-promise": "~3", "uuid": "~3", diff --git a/test/integration/api/submissions.js b/test/integration/api/submissions.js index 852d078d3..9d75ba995 100644 --- a/test/integration/api/submissions.js +++ b/test/integration/api/submissions.js @@ -5,7 +5,7 @@ const { sql } = require('slonik'); const { createReadStream, readFileSync } = require('fs'); const { testService } = require('../setup'); const testData = require('../../data/xml'); -const { zipStreamToFiles } = require('../../util/zip'); +const { pZipStreamToFiles } = require('../../util/zip'); const { Form } = require(appRoot + '/lib/model/frames'); const { exhaust } = require(appRoot + '/lib/worker/worker'); @@ -1186,12 +1186,11 @@ describe('api: /forms/:id/submissions', () => { })))); it('should return the csv header even if there is no data', testService((service) => - service.login('alice', (asAlice) => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => { + service.login('alice', (asAlice) => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n'); - done(); - }))))); + })))); it('should return a zipfile with the relevant data', testService((service) => service.login('alice', (asAlice) => @@ -1207,12 +1206,11 @@ describe('api: /forms/:id/submissions', () => { .send(testData.instances.simple.three) .set('Content-Type', 'text/xml') .expect(200)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); result['simple.csv'].should.be.a.SimpleCsv(); - done(); - })))))); + }))))); it('should include all repeat rows @slow', testService((service) => service.login('alice', (asAlice) => @@ -1236,13 +1234,12 @@ describe('api: /forms/:id/submissions', () => { `) .set('Content-Type', 'text/xml') .expect(200))) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/single-repeat-1-instance-10qs/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/single-repeat-1-instance-10qs/submissions.csv.zip')) + .then((result) => { result.filenames.should.eql([ 'single-repeat-1-instance-10qs.csv', 'single-repeat-1-instance-10qs-repeat.csv' ]); result['single-repeat-1-instance-10qs.csv'].split('\n').length.should.equal(52); result['single-repeat-1-instance-10qs-repeat.csv'].split('\n').length.should.equal(52); - done(); - }))))))); + })))))); it('should not include data from other forms', testService((service) => service.login('alice', (asAlice) => Promise.all([ @@ -1263,8 +1260,8 @@ describe('api: /forms/:id/submissions', () => { .set('Content-Type', 'text/xml') .expect(200)) ]) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const csv = result['simple.csv'].split('\n').map((row) => row.split(',')); csv.length.should.equal(4); // header + 2 data rows + newline @@ -1274,8 +1271,7 @@ describe('api: /forms/:id/submissions', () => { csv[2].shift().should.be.an.recentIsoDate(); csv[2].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]); csv[3].should.eql([ '' ]); - done(); - })))))); + }))))); it('should return a submitter-filtered zipfile with the relevant data', testService((service) => service.login('alice', (asAlice) => @@ -1292,15 +1288,14 @@ describe('api: /forms/:id/submissions', () => { .send(testData.instances.simple.three) .set('Content-Type', 'text/xml') .expect(200)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/submitterId eq 5'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/submitterId eq 5')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const lines = result['simple.csv'].split('\n'); lines.length.should.equal(4); lines[1].endsWith(',three,Chelsea,38,three,5,Alice,0,0,,,,0,').should.equal(true); lines[2].endsWith(',one,Alice,30,one,5,Alice,0,0,,,,0,').should.equal(true); - done(); - }))))))); + })))))); it('should return a review state-filtered zipfile with the relevant data', testService((service) => service.login('alice', (asAlice) => @@ -1319,15 +1314,14 @@ describe('api: /forms/:id/submissions', () => { .send(testData.instances.simple.three) .set('Content-Type', 'text/xml') .expect(200)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/reviewState eq null'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/reviewState eq null')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const lines = result['simple.csv'].split('\n'); lines.length.should.equal(4); lines[1].endsWith(',three,Chelsea,38,three,5,Alice,0,0,,,,0,').should.equal(true); lines[2].endsWith(',one,Alice,30,one,5,Alice,0,0,,,,0,').should.equal(true); - done(); - })))))); + }))))); it('should return a submissionDate-filtered zipfile with the relevant data', testService((service, { run }) => service.login('alice', (asAlice) => @@ -1341,14 +1335,13 @@ describe('api: /forms/:id/submissions', () => { .send(testData.instances.simple.two) .set('Content-Type', 'text/xml') .expect(200)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=year(__system/submissionDate) eq 2010'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=year(__system/submissionDate) eq 2010')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const lines = result['simple.csv'].split('\n'); lines.length.should.equal(3); lines[1].endsWith(',one,Alice,30,one,5,Alice,0,0,,,,0,').should.equal(true); - done(); - }))))))); + })))))); it('should return an updatedAt-filtered zipfile with the relevant data', testService((service) => service.login('alice', (asAlice) => @@ -1363,14 +1356,13 @@ describe('api: /forms/:id/submissions', () => { .then(() => asAlice.patch('/v1/projects/1/forms/simple/submissions/two') .send({ reviewState: 'approved' }) .expect(200)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/updatedAt eq null'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?$filter=__system/updatedAt eq null')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const lines = result['simple.csv'].split('\n'); lines.length.should.equal(3); lines[1].endsWith(',one,Alice,30,one,5,Alice,0,0,,,,0,').should.equal(true); - done(); - })))))); + }))))); it('should return a zipfile with the relevant attachments', testService((service) => service.login('alice', (asAlice) => @@ -1388,8 +1380,8 @@ describe('api: /forms/:id/submissions', () => { .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' }) .attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' }) .expect(201)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'binaryType.csv', 'media/my_file1.mp4', @@ -1404,9 +1396,7 @@ describe('api: /forms/:id/submissions', () => { csv[0].should.equal('SubmissionDate,meta-instanceID,file1,file2,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion'); csv[1].should.endWith(',both,my_file1.mp4,here_is_file2.jpg,both,5,Alice,2,2,,,,0,'); csv.length.should.equal(3); // newline at end - - done(); - }))))))); + })))))); it('should filter attachments by the query', testService((service) => service.login('alice', (asAlice) => @@ -1425,14 +1415,13 @@ describe('api: /forms/:id/submissions', () => { .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.two), { filename: 'data.xml' }) .attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' }) .expect(201)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip?$filter=__system/submitterId eq 5'), (result) => { - result.filenames.should.eql([ - 'binaryType.csv', - 'media/my_file1.mp4' - ]); - done(); - })))))))); + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip?$filter=__system/submitterId eq 5')) + .then((result) => { + result.filenames.should.eql([ + 'binaryType.csv', + 'media/my_file1.mp4' + ]); + }))))))); it('should list the original submitted form version per submission', testService((service) => service.login('alice', (asAlice) => @@ -1461,15 +1450,14 @@ describe('api: /forms/:id/submissions', () => { .send(testData.instances.simple.three.replace('id="simple"', 'id="simple" version="updated"')) .set('Content-Type', 'text/xml') .expect(200)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const lines = result['simple.csv'].split('\n'); lines[1].endsWith('0,updated').should.equal(true); lines[2].endsWith('0,').should.equal(true); lines[3].endsWith('1,').should.equal(true); - done(); - })))))); + }))))); it('should split select multiple values if ?splitSelectMultiples=true', testService((service, container) => service.login('alice', (asAlice) => @@ -1483,8 +1471,8 @@ describe('api: /forms/:id/submissions', () => { .set('Content-Type', 'application/xml') .send(testData.instances.selectMultiple.two)) .then(() => exhaust(container)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true')) + .then((result) => { result.filenames.should.containDeep([ 'selectMultiple.csv' ]); const lines = result['selectMultiple.csv'].split('\n'); lines[0].should.equal('SubmissionDate,q1,q1/a,q1/b,g1-q2,g1-q2/m,g1-q2/x,g1-q2/y,g1-q2/z,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion'); @@ -1492,8 +1480,7 @@ describe('api: /forms/:id/submissions', () => { .should.equal(',b,0,1,m x,1,1,0,0,two,5,Alice,0,0,,,,0,'); lines[2].slice('yyyy-mm-ddThh:mm:ss._msZ'.length) .should.equal(',a b,1,1,x y z,0,1,1,1,one,5,Alice,0,0,,,,0,'); - done(); - })))))); + }))))); it('should omit multiples it does not know about', testService((service, container) => service.login('alice', (asAlice) => @@ -1506,8 +1493,8 @@ describe('api: /forms/:id/submissions', () => { .then(() => asAlice.post('/v1/projects/1/forms/selectMultiple/submissions') .set('Content-Type', 'application/xml') .send(testData.instances.selectMultiple.two)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true')) + .then((result) => { result.filenames.should.containDeep([ 'selectMultiple.csv' ]); const lines = result['selectMultiple.csv'].split('\n'); lines[0].should.equal('SubmissionDate,q1,g1-q2,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion'); @@ -1515,8 +1502,7 @@ describe('api: /forms/:id/submissions', () => { .should.equal(',b,m x,two,5,Alice,0,0,,,,0,'); lines[2].slice('yyyy-mm-ddThh:mm:ss._msZ'.length) .should.equal(',a b,x y z,one,5,Alice,0,0,,,,0,'); - done(); - })))))); + }))))); it('should split select multiples and filter given both options', testService((service, container) => service.login('alice', (asAlice) => @@ -1532,15 +1518,14 @@ describe('api: /forms/:id/submissions', () => { .then(() => asAlice.patch('/v1/projects/1/forms/selectMultiple/submissions/two') .send({ reviewState: 'approved' })) .then(() => exhaust(container)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true&$filter=__system/reviewState eq null'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true&$filter=__system/reviewState eq null')) + .then((result) => { const lines = result['selectMultiple.csv'].split('\n'); lines.length.should.equal(3); lines[1].should.containEql(',one,'); lines[1].should.not.containEql('two'); lines[2].should.equal(''); - done(); - })))))); + }))))); it('should export deleted fields and values if ?deletedFields=true', testService((service) => service.login('alice', (asAlice) => @@ -1573,8 +1558,8 @@ describe('api: /forms/:id/submissions', () => { .then(() => asAlice.post('/v1/projects/1/forms/simple/submissions') .set('Content-Type', 'application/xml') .send(testData.instances.simple.three.replace('id="simple"', 'id="simple" version="2"'))) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?deletedFields=true'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?deletedFields=true')) + .then((result) => { result.filenames.should.containDeep([ 'simple.csv' ]); const lines = result['simple.csv'].split('\n'); lines[0].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion'); @@ -1584,8 +1569,7 @@ describe('api: /forms/:id/submissions', () => { .should.equal(',two,Bob,34,two,5,Alice,0,0,,,,0,'); lines[3].slice('yyyy-mm-ddThh:mm:ss._msZ'.length) .should.equal(',one,Alice,30,one,5,Alice,0,0,,,,0,'); - done(); - })))))); + }))))); it('should skip attachments if ?attachments=false is given', testService((service) => service.login('alice', (asAlice) => @@ -1603,15 +1587,14 @@ describe('api: /forms/:id/submissions', () => { .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' }) .attach('here_is_file2.jpg', Buffer.from('this is test file two'), { filename: 'here_is_file2.jpg' }) .expect(201)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip?attachments=false'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip?attachments=false')) + .then((result) => { result.filenames.should.containDeep([ 'binaryType.csv' ]); should.not.exist(result['media/my_file1.mp4']); should.not.exist(result['media/here_is_file2.jpg']); - done(); - }))))))); + })))))); it('should give the appropriate filename if ?attachments=false is given', testService((service) => service.login('alice', (asAlice) => @@ -1636,12 +1619,11 @@ describe('api: /forms/:id/submissions', () => { .set('X-OpenRosa-Version', '1.0') .attach('xml_submission_file', Buffer.from(testData.instances.simple.one), { filename: 'data.xml' }) .expect(201) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?groupPaths=false'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip?groupPaths=false')) + .then((result) => { const csv = result['simple.csv'].split('\n'); csv[0].should.equal('SubmissionDate,instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion'); - done(); - })))))); + }))))); it('should split select AND omit group paths given both options', testService((service, container) => service.login('alice', (asAlice) => @@ -1655,8 +1637,8 @@ describe('api: /forms/:id/submissions', () => { .set('Content-Type', 'application/xml') .send(testData.instances.selectMultiple.two)) .then(() => exhaust(container)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true&groupPaths=false'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/submissions.csv.zip?splitSelectMultiples=true&groupPaths=false')) + .then((result) => { result.filenames.should.containDeep([ 'selectMultiple.csv' ]); const lines = result['selectMultiple.csv'].split('\n'); lines[0].should.equal('SubmissionDate,q1,q1/a,q1/b,q2,q2/m,q2/x,q2/y,q2/z,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion'); @@ -1664,8 +1646,7 @@ describe('api: /forms/:id/submissions', () => { .should.equal(',b,0,1,m x,1,1,0,0,two,5,Alice,0,0,,,,0,'); lines[2].slice('yyyy-mm-ddThh:mm:ss._msZ'.length) .should.equal(',a b,1,1,x y z,0,1,1,1,one,5,Alice,0,0,,,,0,'); - done(); - })))))); + }))))); it('should properly count present attachments', testService((service) => service.login('alice', (asAlice) => @@ -1678,8 +1659,8 @@ describe('api: /forms/:id/submissions', () => { .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.both), { filename: 'data.xml' }) .attach('my_file1.mp4', Buffer.from('this is test file one'), { filename: 'my_file1.mp4' }) .expect(201) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/binaryType/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'binaryType.csv', 'media/my_file1.mp4' @@ -1691,8 +1672,7 @@ describe('api: /forms/:id/submissions', () => { csv[1].should.endWith(',both,my_file1.mp4,here_is_file2.jpg,both,5,Alice,1,2,,,,0,'); csv.length.should.equal(3); // newline at end - done(); - }))))))); + })))))); it('should return worker-processed consolidated client audit log attachments', testService((service, container) => service.login('alice', (asAlice) => @@ -1711,8 +1691,8 @@ describe('api: /forms/:id/submissions', () => { .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) .expect(201)) .then(() => exhaust(container)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -1730,9 +1710,7 @@ two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); - - done(); - }))) + })) .then(() => container.oneFirst(sql`select count(*) from client_audits`) .then((count) => { count.should.equal(8); }))))); @@ -1752,8 +1730,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' }) .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) .expect(201)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -1771,9 +1749,7 @@ two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); - - done(); - })))))); + }))))); it('should return consolidated client audit log filtered by user', testService((service, container) => service.login('alice', (asAlice) => @@ -1792,8 +1768,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' }) .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) .expect(201)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip?$filter=__system/submitterId eq 5'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip?$filter=__system/submitterId eq 5')) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -1808,8 +1784,7 @@ one,d,/data/d,2000-01-01T00:10,,10,11,12,gg, one,e,/data/e,2000-01-01T00:11,,,,,hh,ii `); - done(); - }))))))); + })))))); it('should return consolidated client audit log filtered by review state', testService((service, container) => service.login('alice', (asAlice) => @@ -1830,8 +1805,8 @@ one,e,/data/e,2000-01-01T00:11,,,,,hh,ii .attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' }) .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) .expect(201)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip?$filter=__system/reviewState eq \'approved\''), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip?$filter=__system/reviewState eq \'approved\'')) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -1846,8 +1821,7 @@ one,d,/data/d,2000-01-01T00:10,,10,11,12,gg, one,e,/data/e,2000-01-01T00:11,,,,,hh,ii `); - done(); - })))))); + }))))); it('should return the latest attached audit log after openrosa replace', testService((service) => service.login('alice', (asAlice) => @@ -1867,8 +1841,8 @@ one,e,/data/e,2000-01-01T00:11,,,,,hh,ii .expect(201)) .then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -1881,8 +1855,7 @@ one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); - done(); - }))))))); + })))))); it('should return the latest attached audit log after REST replace', testService((service) => service.login('alice', (asAlice) => @@ -1901,8 +1874,8 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .expect(200)) .then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -1915,8 +1888,7 @@ one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); - done(); - }))))))); + })))))); it('should tolerate differences in line lengths', testService((service) => service.login('alice', (asAlice) => @@ -1931,8 +1903,8 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .expect(201)) .then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -1946,9 +1918,7 @@ one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff one,d,/data/d,2000-01-01T00:10,,10,11,12,, one,e,/data/e,2000-01-01T00:11,,,,,hh,ii `); - - done(); - }))))))); + })))))); it('should tolerate quote inside unquoted field of client audit log', testService((service) => service.login('alice', (asAlice) => @@ -1963,8 +1933,8 @@ one,e,/data/e,2000-01-01T00:11,,,,,hh,ii .expect(201)) .then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -1978,9 +1948,7 @@ one,c,/data/c,2000-01-01T00:03,2000-01-01T00:04,7,8,9,ee,ff one,d,/data/d,2000-01-01T00:10,,10,11,12,"g""g", one,e,/data/e,2000-01-01T00:11,,,,,hh,ii `); - - done(); - }))))))); + })))))); context('versioning', () => { const withClientAuditIds = (deprecatedId, instanceId) => testData.instances.clientAudits.one @@ -2004,8 +1972,8 @@ one,e,/data/e,2000-01-01T00:11,,,,,hh,ii .expect(201)) .then(() => asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/audits/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -2017,9 +1985,7 @@ one,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb one,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); - - done(); - }))))))); + })))))); }); it('should log the action in the audit log', testService((service) => @@ -2166,8 +2132,8 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .expect(200)) .then(() => asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'simple.csv' ]); const csv = result['simple.csv'].split('\n').map((row) => row.split(',')); @@ -2175,9 +2141,7 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff csv[0].should.eql([ 'SubmissionDate', 'meta-instanceID', 'name', 'age', 'KEY', 'SubmitterID', 'SubmitterName', 'AttachmentsPresent', 'AttachmentsExpected', 'Status', 'ReviewState', 'DeviceID', 'Edits', 'FormVersion' ]); csv[1].shift().should.be.an.recentIsoDate(); csv[1].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]); - - done(); - }))))))); + })))))); it('should not include draft submissions in nondraft csvzip', testService((service) => service.login('alice', (asAlice) => @@ -2189,13 +2153,12 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .expect(200)) .then(() => asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'simple.csv' ]); result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n'); - done(); - }))))))); + })))))); it('should not carry draft submissions forward to the published version upon publish', testService((service) => service.login('alice', (asAlice) => @@ -2209,13 +2172,12 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .expect(200)) .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'simple.csv' ]); result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n'); - done(); - }))))))); + })))))); it('should not carry over drafts when a draft is replaced', testService((service) => service.login('alice', (asAlice) => @@ -2229,13 +2191,12 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .expect(200)) .then(() => asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'simple.csv' ]); result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n'); - done(); - }))))))); + })))))); it('should not resurface drafts when a draft is recreated', testService((service) => service.login('alice', (asAlice) => @@ -2251,13 +2212,12 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .expect(200)) .then(() => asAlice.get('/v1/projects/1/forms/simple/draft/submissions.csv.zip') .expect(200) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/simple/submissions.csv.zip')) + .then((result) => { result.filenames.should.containDeep([ 'simple.csv' ]); result['simple.csv'].should.equal('SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion\n'); - done(); - }))))))); + })))))); it('should not log the action in the audit log', testService((service) => service.login('alice', (asAlice) => @@ -2288,8 +2248,8 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .set('X-OpenRosa-Version', '1.0') .attach('xml_submission_file', Buffer.from(testData.instances.selectMultiple.two), { filename: 'data.xml' })))) .then(() => exhaust(container)) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/draft/submissions.csv.zip?splitSelectMultiples=true'), (result) => { + .then(() => pZipStreamToFiles(asAlice.get('/v1/projects/1/forms/selectMultiple/draft/submissions.csv.zip?splitSelectMultiples=true')) + .then((result) => { result.filenames.should.containDeep([ 'selectMultiple.csv' ]); const lines = result['selectMultiple.csv'].split('\n'); lines[0].should.equal('SubmissionDate,q1,q1/a,q1/b,g1-q2,g1-q2/m,g1-q2/x,g1-q2/y,g1-q2/z,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion'); @@ -2297,8 +2257,7 @@ one,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .should.equal(',b,0,1,m x,1,1,0,0,two,,,0,0,,,,0,'); lines[2].slice('yyyy-mm-ddThh:mm:ss._msZ'.length) .should.equal(',a b,1,1,x y z,0,1,1,1,one,,,0,0,,,,0,'); - done(); - })))))); + }))))); }); describe('GET', () => { diff --git a/test/integration/other/encryption.js b/test/integration/other/encryption.js index d134548db..1f03228a5 100644 --- a/test/integration/other/encryption.js +++ b/test/integration/other/encryption.js @@ -5,7 +5,7 @@ const { sql } = require('slonik'); const { toText } = require('streamtest').v2; const { testService, testContainerFullTrx, testContainer } = require(appRoot + '/test/integration/setup'); const testData = require(appRoot + '/test/data/xml'); -const { zipStreamToFiles } = require(appRoot + '/test/util/zip'); +const { pZipStreamToFiles } = require(appRoot + '/test/util/zip'); const { Form, Key, Submission } = require(appRoot + '/lib/model/frames'); const { mapSequential } = require(appRoot + '/test/util/util'); const { exhaust } = require(appRoot + '/lib/worker/worker'); @@ -212,12 +212,11 @@ describe('managed encryption', () => { .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys') .expect(200) .then(({ body }) => body[0].id)) - .then((keyId) => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => { + .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`)) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); result['simple.csv'].should.be.an.EncryptedSimpleCsv(); - done(); - })))))); + }))))); it('should decrypt to CSV successfully as a direct root table', testService((service) => service.login('alice', (asAlice) => @@ -251,14 +250,13 @@ describe('managed encryption', () => { .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys') .expect(200) .then(({ body }) => body[0].id)) - .then((keyId) => new Promise((done) => - zipStreamToFiles(asAlice.post(`/v1/projects/1/forms/simple/submissions.csv.zip`) + .then((keyId) => pZipStreamToFiles(asAlice.post(`/v1/projects/1/forms/simple/submissions.csv.zip`) .send(`${keyId}=supersecret`) - .set('Content-Type', 'application/x-www-form-urlencoded'), (result) => { + .set('Content-Type', 'application/x-www-form-urlencoded')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); result['simple.csv'].should.be.an.EncryptedSimpleCsv(); - done(); - })))))); + }))))); it('should decrypt over cookie auth with passphrases provided via url-encoded POST body', testService((service) => service.login('alice', (asAlice) => @@ -280,16 +278,15 @@ describe('managed encryption', () => { .expect(200) .then(({ body }) => body) ])) - .then(([ keyId, session ]) => new Promise((done) => - zipStreamToFiles(service.post(`/v1/projects/1/forms/simple/submissions.csv.zip`) + .then(([ keyId, session ]) => pZipStreamToFiles(service.post(`/v1/projects/1/forms/simple/submissions.csv.zip`) .send(`${keyId}=supersecret&__csrf=${session.csrf}`) .set('Cookie', `__Host-session=${session.token}`) .set('X-Forwarded-Proto', 'https') - .set('Content-Type', 'application/x-www-form-urlencoded'), (result) => { + .set('Content-Type', 'application/x-www-form-urlencoded')) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); result['simple.csv'].should.be.an.EncryptedSimpleCsv(); - done(); - })))))); + }))))); it('should decrypt with passphrases provide via JSON POST body', testService((service) => service.login('alice', (asAlice) => @@ -305,13 +302,12 @@ describe('managed encryption', () => { .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys') .expect(200) .then(({ body }) => body[0].id)) - .then((keyId) => new Promise((done) => - zipStreamToFiles(asAlice.post(`/v1/projects/1/forms/simple/submissions.csv.zip`) - .send({ [keyId]: 'supersecret' }), (result) => { + .then((keyId) => pZipStreamToFiles(asAlice.post(`/v1/projects/1/forms/simple/submissions.csv.zip`) + .send({ [keyId]: 'supersecret' })) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); result['simple.csv'].should.be.an.EncryptedSimpleCsv(); - done(); - })))))); + }))))); it('should decrypt attached files successfully', testService((service) => service.login('alice', (asAlice) => @@ -326,16 +322,15 @@ describe('managed encryption', () => { .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys') .expect(200) .then(({ body }) => body[0].id)) - .then((keyId) => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => { + .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`)) + .then((result) => { result.filenames.length.should.equal(4); result.filenames.should.containDeep([ 'simple.csv', 'media/alpha', 'media/beta', 'media/charlie' ]); result['media/alpha'].should.equal('hello this is file alpha'); result['media/beta'].should.equal('and beta'); result['media/charlie'].should.equal('file charlie is right here'); - done(); - })))))); + }))))); it('should strip .enc suffix from decrypted attachments', testService((service) => service.login('alice', (asAlice) => @@ -349,14 +344,13 @@ describe('managed encryption', () => { .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys') .expect(200) .then(({ body }) => body[0].id)) - .then((keyId) => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => { + .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`)) + .then((result) => { result.filenames.length.should.equal(2); result.filenames.should.containDeep([ 'simple.csv', 'media/testfile.jpg' ]); result['media/testfile.jpg'].should.equal('hello this is a suffixed file'); - done(); - }))))))); + })))))); it('should decrypt client audit log attachments', testService((service, container) => service.login('alice', (asAlice) => @@ -396,8 +390,8 @@ describe('managed encryption', () => { .then(() => asAlice.get('/v1/projects/1/forms/audits/submissions/keys') .expect(200) .then(({ body }) => body[0].id)) - .then((keyId) => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/audits/submissions.csv.zip?${keyId}=supersecret`), (result) => { + .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/audits/submissions.csv.zip?${keyId}=supersecret`)) + .then((result) => { result.filenames.should.containDeep([ 'audits.csv', 'media/audit.csv', @@ -415,9 +409,7 @@ two,f,/data/f,2000-01-01T00:04,2000-01-01T00:05,-1,-2,,aa,bb two,g,/data/g,2000-01-01T00:05,2000-01-01T00:06,-3,-4,,cc,dd two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff `); - - done(); - })))))); + }))))); it('should handle mixed [plaintext/encrypted] attachments (not decrypting)', testService((service) => service.login('alice', (asAlice) => @@ -439,14 +431,13 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .expect(200) .then(({ text }) => sendEncrypted(asAlice, extractVersion(text), extractPubkey(text))) .then((send) => send(testData.instances.binaryType.two, { 'here_is_file2.jpg': 'file two you cant see' }))) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/binaryType/submissions.csv.zip`), (result) => { + .then(() => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/binaryType/submissions.csv.zip`)) + .then((result) => { result.filenames.length.should.equal(2); result.filenames.should.containDeep([ 'binaryType.csv', 'media/my_file1.mp4' ]); result['media/my_file1.mp4'].should.equal('this is file one'); - done(); - })))))); + }))))); it('should handle mixed [plaintext/encrypted] attachments (decrypting)', testService((service) => service.login('alice', (asAlice) => @@ -471,15 +462,14 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .then(() => asAlice.get('/v1/projects/1/forms/binaryType/submissions/keys') .expect(200) .then(({ body }) => body[0].id)) - .then((keyId) => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/binaryType/submissions.csv.zip?${keyId}=supersecret`), (result) => { + .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/binaryType/submissions.csv.zip?${keyId}=supersecret`)) + .then((result) => { result.filenames.length.should.equal(3); result.filenames.should.containDeep([ 'binaryType.csv', 'media/my_file1.mp4', 'media/here_is_file2.jpg' ]); result['media/my_file1.mp4'].should.equal('this is file one'); result['media/here_is_file2.jpg'].should.equal('file two you can see'); - done(); - })))))); + }))))); it('should handle mixed[plaintext/encrypted] formdata (decrypting)', testService((service) => service.login('alice', (asAlice) => @@ -498,8 +488,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys') .expect(200) .then(({ body }) => body[0].id)) - .then((keyId) => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => { + .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`)) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const csv = result['simple.csv'].split('\n').map((row) => row.split(',')); csv.length.should.equal(5); // header + 3 data rows + newline @@ -513,8 +503,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff csv[3].shift().should.be.an.recentIsoDate(); csv[3].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]); csv[4].should.eql([ '' ]); - done(); - }))))))); + })))))); it('should handle mixed[plaintext/encrypted] formdata (not decrypting)', testService((service, { Project, FormPartial }) => service.login('alice', (asAlice) => @@ -530,8 +519,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .then(({ text }) => sendEncrypted(asAlice, extractVersion(text), extractPubkey(text))) .then((send) => send(testData.instances.simple.two) .then(() => send(testData.instances.simple.three)))) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip`), (result) => { + .then(() => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip`)) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const csv = result['simple.csv'].split('\n').map((row) => row.split(',')); @@ -546,8 +535,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff csv[3].shift().should.be.an.recentIsoDate(); csv[3].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]); csv[4].should.eql([ '' ]); - done(); - }))))))); + })))))); // we have to sort of cheat at this to get two different managed keys in effect. it('should handle mixed[managedA/managedB] formdata (decrypting)', testService((service, { Forms, Projects }) => @@ -585,8 +573,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys') .expect(200) .then(({ body }) => body.map((key) => key.id))) - .then((keyIds) => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyIds[1]}=supersecret&${keyIds[0]}=superdupersecret`), (result) => { + .then((keyIds) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyIds[1]}=supersecret&${keyIds[0]}=superdupersecret`)) + .then((result) => { const csv = result['simple.csv'].split('\n').map((row) => row.split(',')); csv.length.should.equal(5); // header + 3 data rows + newline csv[0].should.eql([ 'SubmissionDate', 'meta-instanceID', 'name', 'age', 'KEY', 'SubmitterID', 'SubmitterName', 'AttachmentsPresent', 'AttachmentsExpected', 'Status', 'ReviewState', 'DeviceID', 'Edits', 'FormVersion' ]); @@ -600,8 +588,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff csv[3].pop().should.match(/^\[encrypted:........\]$/); csv[3].should.eql([ 'one','Alice','30','one','5','Alice','1','1','','','','0' ]); csv[4].should.eql([ '' ]); - done(); - })))))); + }))))); it('should handle mixed [plaintext/missing-encrypted-xml] formdata (decrypting)', testService((service, { Project, FormPartial }) => service.login('alice', (asAlice) => @@ -622,8 +609,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .then(() => asAlice.get('/v1/projects/1/forms/simple/submissions/keys') .expect(200) .then(({ body }) => body[0].id)) - .then((keyId) => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`), (result) => { + .then((keyId) => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip?${keyId}=supersecret`)) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const csv = result['simple.csv'].split('\n').map((row) => row.split(',')); @@ -635,8 +622,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff csv[2].shift().should.be.an.recentIsoDate(); csv[2].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]); csv[3].should.eql([ '' ]); - done(); - }))))))); + })))))); it('should handle mixed [plaintext/missing-encrypted-xml] formdata (not decrypting)', testService((service, { Project, FormPartial }) => service.login('alice', (asAlice) => @@ -654,8 +640,8 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff .send(envelope) .set('Content-Type', 'text/xml') .expect(200))) - .then(() => new Promise((done) => - zipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip`), (result) => { + .then(() => pZipStreamToFiles(asAlice.get(`/v1/projects/1/forms/simple/submissions.csv.zip`)) + .then((result) => { result.filenames.should.eql([ 'simple.csv' ]); const csv = result['simple.csv'].split('\n').map((row) => row.split(',')); @@ -667,8 +653,7 @@ two,h,/data/h,2000-01-01T00:06,2000-01-01T00:07,-5,-6,,ee,ff csv[2].shift().should.be.an.recentIsoDate(); csv[2].should.eql([ 'one','Alice','30','one','5','Alice','0','0','','','','0','' ]); csv[3].should.eql([ '' ]); - done(); - }))))))); + })))))); }); }); diff --git a/test/unit/data/attachments.js b/test/unit/data/attachments.js index d0b7297cd..c46dee12e 100644 --- a/test/unit/data/attachments.js +++ b/test/unit/data/attachments.js @@ -12,7 +12,9 @@ describe('.zip attachments streaming', () => { { row: { instanceId: 'subone', name: 'secondfile.ext', content: 'this is my second file' } }, { row: { instanceId: 'subtwo', name: 'thirdfile.ext', content: 'this is my third file' } } ]); - zipStreamToFiles(zipStreamFromParts(streamAttachments(inStream)), (result) => { + zipStreamToFiles(zipStreamFromParts(() => streamAttachments(inStream)), (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'media/firstfile.ext', 'media/secondfile.ext', @@ -33,7 +35,9 @@ describe('.zip attachments streaming', () => { { row: { instanceId: 'subone', name: '../secondfile.ext', content: 'this is my second file' } }, { row: { instanceId: 'subone', name: './.secondfile.ext', content: 'this is my duplicate second file' } }, ]); - zipStreamToFiles(zipStreamFromParts(streamAttachments(inStream)), (result) => { + zipStreamToFiles(zipStreamFromParts(() => streamAttachments(inStream)), (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'media/firstfile.ext', 'media/..secondfile.ext', @@ -48,7 +52,9 @@ describe('.zip attachments streaming', () => { const inStream = streamTest.fromObjects([ { row: { instanceId: 'subone', name: 'firstfile.ext.enc', content: 'this is my first file' } } ]); - zipStreamToFiles(zipStreamFromParts(streamAttachments(inStream)), (result) => { + zipStreamToFiles(zipStreamFromParts(() => streamAttachments(inStream)), (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'media/firstfile.ext.enc' ]); done(); }); @@ -58,7 +64,9 @@ describe('.zip attachments streaming', () => { const inStream = streamTest.fromObjects([ { row: { instanceId: 'subone', name: 'firstfile.ext.enc', content: 'this is my first file' } } ]); - zipStreamToFiles(zipStreamFromParts(streamAttachments(inStream, () => {})), (result) => { + zipStreamToFiles(zipStreamFromParts(() => streamAttachments(inStream, () => {})), (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'media/firstfile.ext' ]); done(); }); diff --git a/test/unit/data/briefcase.js b/test/unit/data/briefcase.js index 5c43939c6..8da16ce27 100644 --- a/test/unit/data/briefcase.js +++ b/test/unit/data/briefcase.js @@ -28,7 +28,7 @@ const withAttachments = (present, expected, row) => ({ ...row, aux: { ...row.aux const callAndParse = (inStream, formXml, xmlFormId, callback) => { fieldsFor(formXml).then((fields) => { - zipStreamToFiles(zipStreamFromParts(streamBriefcaseCsvs(inStream, fields, xmlFormId)), callback); + zipStreamToFiles(zipStreamFromParts(() => streamBriefcaseCsvs(inStream, fields, xmlFormId)), callback); }); }; @@ -60,7 +60,9 @@ describe('.csv.zip briefcase output @slow', () => { instance('three', 'Chelsea38San Francisco, CA') ]); - callAndParse(inStream, formXml, 'mytestform', (result) => { + callAndParse(inStream, formXml, 'mytestform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'mytestform.csv' ]); result['mytestform.csv'].should.equal( `SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -82,7 +84,7 @@ describe('.csv.zip briefcase output @slow', () => { }]); // not hanging is the assertion here: - callAndParse(inStream, testData.forms.simple, 'simple', () => { done(); }); + callAndParse(inStream, testData.forms.simple, 'simple', (err) => { done(err); }); }); it('should attach submitter information if present', (done) => { @@ -111,7 +113,9 @@ describe('.csv.zip briefcase output @slow', () => { withSubmitter(15, 'lito', instance('three', 'Chelsea38San Francisco, CA')) ]); - callAndParse(inStream, formXml, 'mytestform', (result) => { + callAndParse(inStream, formXml, 'mytestform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'mytestform.csv' ]); result['mytestform.csv'].should.equal( `SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -149,7 +153,9 @@ describe('.csv.zip briefcase output @slow', () => { withAttachments(3, 3, instance('three', 'Chelsea38San Francisco, CA')) ]); - callAndParse(inStream, formXml, 'mytestform', (result) => { + callAndParse(inStream, formXml, 'mytestform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'mytestform.csv' ]); result['mytestform.csv'].should.equal( `SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -186,7 +192,9 @@ describe('.csv.zip briefcase output @slow', () => { { ...instance('two', 'Bob34Portland, OR'), reviewState: 'rejected' } ]); - callAndParse(inStream, formXml, 'mytestform', (result) => { + callAndParse(inStream, formXml, 'mytestform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'mytestform.csv' ]); result['mytestform.csv'].should.equal( `SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -221,7 +229,9 @@ describe('.csv.zip briefcase output @slow', () => { { ...instance('one', 'missing'), xml: 'missing', deviceId: 'test device' } ]); - callAndParse(inStream, formXml, 'mytestform', (result) => { + callAndParse(inStream, formXml, 'mytestform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'mytestform.csv' ]); result['mytestform.csv'].should.equal( `SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -255,7 +265,9 @@ describe('.csv.zip briefcase output @slow', () => { data.aux.edit.count = 3; const inStream = streamTest.fromObjects([ data ]); - callAndParse(inStream, formXml, 'mytestform', (result) => { + callAndParse(inStream, formXml, 'mytestform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'mytestform.csv' ]); result['mytestform.csv'].should.equal( `SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -291,7 +303,9 @@ describe('.csv.zip briefcase output @slow', () => { two.aux.exports.formVersion = 'updated'; const inStream = streamTest.fromObjects([ one, two ]); - callAndParse(inStream, formXml, 'mytestform', (result) => { + callAndParse(inStream, formXml, 'mytestform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'mytestform.csv' ]); result['mytestform.csv'].should.equal( `SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -326,7 +340,9 @@ describe('.csv.zip briefcase output @slow', () => { instance('one', '«Alice»30Seattle, WA'), ]); - callAndParse(inStream, formXml, 'mytestform', (result) => { + callAndParse(inStream, formXml, 'mytestform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'mytestform.csv' ]); result['mytestform.csv'].should.equal( `SubmissionDate,name,age,hometown,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -362,7 +378,9 @@ describe('.csv.zip briefcase output @slow', () => { instance('three', 'Chelsea38') ]); - callAndParse(inStream, formXml, 'mytestform', (result) => { + callAndParse(inStream, formXml, 'mytestform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'mytestform.csv' ]); result['mytestform.csv'].should.equal( `SubmissionDate,name,age,location-Latitude,location-Longitude,location-Altitude,location-Accuracy,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -381,7 +399,9 @@ describe('.csv.zip briefcase output @slow', () => { ]); fieldsFor(testData.forms.selectMultiple).then((fields) => { - zipStreamToFiles(zipStreamFromParts(streamBriefcaseCsvs(inStream, fields, 'selectMultiple', { '/q1': [ 'x', 'y', 'z' ], '/g1/q2': [ 'm', 'n' ] })), (result) => { + zipStreamToFiles(zipStreamFromParts(() => streamBriefcaseCsvs(inStream, fields, 'selectMultiple', { '/q1': [ 'x', 'y', 'z' ], '/g1/q2': [ 'm', 'n' ] })), (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'selectMultiple.csv' ]); result['selectMultiple.csv'].should.equal( `SubmissionDate,q1,q1/x,q1/y,q1/z,g1-q2,g1-q2/m,g1-q2/n,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -429,7 +449,9 @@ describe('.csv.zip briefcase output @slow', () => { instance('three', 'threeChelseaHouse
San Francisco, CA99 Mission Ave
'), ]); - callAndParse(inStream, formXml, 'structuredform', (result) => { + callAndParse(inStream, formXml, 'structuredform', (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'structuredform.csv' ]); result['structuredform.csv'].should.equal( `SubmissionDate,meta-instanceID,name,home-type,home-address-street,home-address-city,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -478,7 +500,9 @@ describe('.csv.zip briefcase output @slow', () => { ]); fieldsFor(formXml).then((fields) => { - zipStreamToFiles(zipStreamFromParts(streamBriefcaseCsvs(inStream, fields, 'structuredform', undefined, undefined, false, { groupPaths: false })), (result) => { + zipStreamToFiles(zipStreamFromParts(() => streamBriefcaseCsvs(inStream, fields, 'structuredform', undefined, undefined, false, { groupPaths: false })), (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'structuredform.csv' ]); result['structuredform.csv'].should.equal( `SubmissionDate,instanceID,name,type,street,city,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -498,7 +522,9 @@ describe('.csv.zip briefcase output @slow', () => { ]); fieldsFor(testData.forms.selectMultiple).then((fields) => { - zipStreamToFiles(zipStreamFromParts(streamBriefcaseCsvs(inStream, fields, 'selectMultiple', { '/q1': [ 'x', 'y', 'z' ], '/g1/q2': [ 'm', 'n' ] }, undefined, false, { groupPaths: false })), (result) => { + zipStreamToFiles(zipStreamFromParts(() => streamBriefcaseCsvs(inStream, fields, 'selectMultiple', { '/q1': [ 'x', 'y', 'z' ], '/g1/q2': [ 'm', 'n' ] }, undefined, false, { groupPaths: false })), (err, result) => { + if(err) return done(err); + result.filenames.should.eql([ 'selectMultiple.csv' ]); result['selectMultiple.csv'].should.equal( `SubmissionDate,q1,q1/x,q1/y,q1/z,q2,q2/m,q2/n,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -565,7 +591,9 @@ describe('.csv.zip briefcase output @slow', () => { instance('three', 'threeChelsea38Candace2'), ]); - callAndParse(inStream, formXml, 'singlerepeat', (result) => { + callAndParse(inStream, formXml, 'singlerepeat', (err, result) => { + if(err) return done(err); + result.filenames.should.containDeep([ 'singlerepeat.csv', 'singlerepeat-child.csv' ]); result['singlerepeat.csv'].should.equal( `SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -627,7 +655,9 @@ Candace,2,three,three/children/child[1] const inStream = streamTest.fromObjects( (new Array(127)).fill(null).map(_ => instance(uuid(), `${uuid()}${uuid()}${uuid()}`))); - callAndParse(inStream, formXml, 'singlerepeat', (result) => { + callAndParse(inStream, formXml, 'singlerepeat', (err, result) => { + if(err) return done(err); + result.filenames.should.containDeep([ 'singlerepeat.csv', 'singlerepeat-child.csv' ]); result['singlerepeat.csv'].split('\n').length.should.equal(129); result['singlerepeat-child.csv'].split('\n').length.should.equal(129); @@ -702,7 +732,9 @@ Candace,2,three,three/children/child[1] instance('three', 'threeChelsea38CandaceMillennium FalconX-WingPod racer2'), ]); - callAndParse(inStream, formXml, 'multirepeat', (result) => { + callAndParse(inStream, formXml, 'multirepeat', (err, result) => { + if(err) return done(err); + result.filenames.should.containDeep([ 'multirepeat.csv', 'multirepeat-child.csv', 'multirepeat-toy.csv' ]); result['multirepeat.csv'].should.equal( `SubmissionDate,meta-instanceID,name,age,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -762,7 +794,9 @@ Pod racer,three/children/child[1],three/children/child[1]/toy[3] instance('one', 'AliceBobChelseaLiving at home') ]); - callAndParse(inStream, formXml, 'pathprefix', (result) => { + callAndParse(inStream, formXml, 'pathprefix', (err, result) => { + if(err) return done(err); + result.filenames.should.containDeep([ 'pathprefix.csv', 'pathprefix-children.csv' ]); result['pathprefix.csv'].should.equal( `SubmissionDate,name,children-status,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -851,7 +885,9 @@ Chelsea,one,one/children[2] aux: { attachment: { present: 0, expected: 0 }, encryption: {}, edit: { count: 0 }, exports: { formVersion: '' } } }]); - callAndParse(inStream, formXml, 'all-data-types', (result) => { + callAndParse(inStream, formXml, 'all-data-types', (err, result) => { + if(err) return done(err); + result.filenames.should.containDeep([ 'all-data-types.csv' ]); result['all-data-types.csv'].should.equal( `SubmissionDate,some_string,some_int,some_decimal,some_date,some_time,some_date_time,some_geopoint-Latitude,some_geopoint-Longitude,some_geopoint-Altitude,some_geopoint-Accuracy,some_geotrace,some_geoshape,some_barcode,meta-instanceID,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -978,7 +1014,9 @@ Chelsea,one,one/children[2] aux: { attachment: { present: 0, expected: 0 }, encryption: {}, edit: { count: 0 }, exports: { formVersion: '' } } }]); - callAndParse(inStream, formXml, 'nested-repeats', (result) => { + callAndParse(inStream, formXml, 'nested-repeats', (err, result) => { + if(err) return done(err); + result.filenames.should.containDeep([ 'nested-repeats.csv', 'nested-repeats-g1.csv', 'nested-repeats-g2.csv', 'nested-repeats-g3.csv' ]); result['nested-repeats.csv'].should.equal( `SubmissionDate,meta-instanceID,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion @@ -1064,7 +1102,9 @@ some text 3.1.4,uuid:0a1b861f-a5fd-4f49-846a-78dcf06cfc1b/g1[3]/g2[1],uuid:0a1b8 instance('three', 'threeChelseaInstantaneous FoodFerrenceMick'), ]); - callAndParse(inStream, formXml, 'ambiguous', (result) => { + callAndParse(inStream, formXml, 'ambiguous', (err, result) => { + if(err) return done(err); + result.filenames.should.containDeep([ 'ambiguous.csv', 'ambiguous-entry~1.csv', 'ambiguous-entry~2.csv' ]); result['ambiguous.csv'].should.equal( `SubmissionDate,meta-instanceID,name,KEY,SubmitterID,SubmitterName,AttachmentsPresent,AttachmentsExpected,Status,ReviewState,DeviceID,Edits,FormVersion diff --git a/test/unit/util/zip.js b/test/unit/util/zip.js index 9d723d8d7..65f056109 100644 --- a/test/unit/util/zip.js +++ b/test/unit/util/zip.js @@ -13,7 +13,9 @@ describe('zipPart streamer', () => { const part = zipPart(); let closed = false; - zipStreamToFiles(zipStreamFromParts(part), (result) => { + zipStreamToFiles(zipStreamFromParts(() => part), (err, result) => { + if(err) return done(err); + closed = true; done(); }); @@ -26,27 +28,26 @@ describe('zipPart streamer', () => { it('should close the archive successfully given no files', (done) => { const part = zipPart(); // no assertions other than verifying that done is called. - zipStreamToFiles(zipStreamFromParts(part), () => done()); + zipStreamToFiles(zipStreamFromParts(() => part), (err) => done(err)); part.finalize(); }); it('should error out the archive if a part pushes an error', (done) => { - const part1 = zipPart(); - const part2 = zipPart(); - const archive = zipStreamFromParts(part1, part2); + const part = zipPart(); + const archive = zipStreamFromParts(() => part); archive.on('error', (err) => { err.message.should.equal('whoops'); done(); }); - part1.append('test 1', { name: 'x/test1.file' }); - part2.error(new Error('whoops')); + part.append('test 1', { name: 'x/test1.file' }); + setTimeout(() => { part.error(new Error('whoops')); }); }); it('should call the given callback only when the file has been added', (done) => { const part = zipPart(); const file = new Readable({ read() {} }); - const archive = zipStreamFromParts(part); + const archive = zipStreamFromParts(() => part); let pushedAll = false; part.append(file, { name: 'file' }, () => { @@ -65,7 +66,7 @@ describe('zipPart streamer', () => { const part = zipPart(); const file1 = new Readable({ read() {} }); const file2 = new Readable({ read() {} }); - const archive = zipStreamFromParts(part); + const archive = zipStreamFromParts(() => part); archive.pipe(createWriteStream('/dev/null')); archive.on('end', () => { @@ -93,7 +94,9 @@ describe('zipPart streamer', () => { const part1 = zipPart(); const part2 = zipPart(); - zipStreamToFiles(zipStreamFromParts(part1, part2), (result) => { + zipStreamToFiles(zipStreamFromParts(() => part1, () => part2), (err, result) => { + if(err) return done(err); + result.filenames.should.containDeep([ 'x/test1.file', 'x/test2.file', @@ -122,7 +125,9 @@ describe('zipPart streamer', () => { const part1 = zipPart(); const part2 = zipPart(); - zipStreamToFiles(zipStreamFromParts(part1, part2), (result) => { + zipStreamToFiles(zipStreamFromParts(() => part1, () => part2), (err, result) => { + if(err) return done(err); + result.filenames.should.containDeep([ 'test1.file', 'test2.file' ]); result['test1.file'].should.equal('test static'); result['test2.file'].should.equal('a!test!stream!'); @@ -145,7 +150,7 @@ describe('zipPart streamer', () => { const part1 = zipPart(); const part2 = zipPart(); - const archive = zipStreamFromParts(part1, part2); + const archive = zipStreamFromParts(() => part1, () => part2); let errCount = 0; archive.on('error', (err) => { errCount += 1; @@ -174,7 +179,7 @@ describe('zipPart streamer', () => { const part1 = zipPart(); const part2 = zipPart(); - const archive = zipStreamFromParts(part1, part2); + const archive = zipStreamFromParts(() => part1, () => part2); archive.on('error', (err) => { err.message.should.equal('whoops'); done(); diff --git a/test/util/zip.js b/test/util/zip.js index c79129fb2..1717f8f1a 100644 --- a/test/util/zip.js +++ b/test/util/zip.js @@ -14,31 +14,38 @@ const streamTest = require('streamtest').v2; // … // } const zipStreamToFiles = (zipStream, callback) => { - tmp.file((_, tmpfile) => { + tmp.file((err, tmpfile) => { + if(err) return callback(err); + const writeStream = createWriteStream(tmpfile); zipStream.pipe(writeStream); zipStream.on('end', () => { setTimeout(() => { - yauzl.open(tmpfile, { autoClose: false }, (_, zipfile) => { + yauzl.open(tmpfile, { autoClose: false }, (err, zipfile) => { + if(err) return callback(err); + const result = { filenames: [] }; let entries = []; let completed = 0; - should.exist(zipfile); zipfile.on('entry', (entry) => entries.push(entry)); zipfile.on('end', () => { if (entries.length === 0) { - callback(result); + callback(null, result); zipfile.close(); } else { entries.forEach((entry) => { result.filenames.push(entry.fileName); - zipfile.openReadStream(entry, (_, resultStream) => { - resultStream.pipe(streamTest.toText((_, contents) => { + zipfile.openReadStream(entry, (err, resultStream) => { + if(err) return callback(err); + + resultStream.pipe(streamTest.toText((err, contents) => { + if(err) return callback(err); + result[entry.fileName] = contents; completed += 1; if (completed === entries.length) { - callback(result); + callback(null, result); zipfile.close(); } })); @@ -52,5 +59,7 @@ const zipStreamToFiles = (zipStream, callback) => { }); }; -module.exports = { zipStreamToFiles }; +const pZipStreamToFiles = (zipStream) => new Promise((resolve, reject) => zipStreamToFiles(zipStream, (err, result) => err ? reject(err) : resolve(result))); + +module.exports = { zipStreamToFiles, pZipStreamToFiles };