Apination Node Stream Utils

Overview

A set of helper utils to work with remote streams

const utils = require('apination-stream-utils');

createReadStream()

loads data from S3.

const TEST_DATA_SRC = 's3://apination-cn-data/staging/cn-example/transactions.json';
const data = [];
const stream = utils.createReadStream(TEST_DATA_SRC);
stream.on('data', chunk => data.push(chunk.toString()));
stream.on('end', () => {
	data.should.have.length(1);
	data[0].should.be.a('String');
	done();
});

createReadArrayStream()

loads JSON array from S3.

const data = [];
const stream = utils.createReadArrayStream(TEST_DATA_SRC);
stream.on('data', chunk => data.push(chunk));
stream.on('end', () => {
	data.should.have.length(2);
	data[0].should.be.an('Object');
	done();
});

createWriteStream()

writes stream to S3.

utils.createReadStream(TEST_DATA_SRC)
	.pipe(utils.createWriteStream(TEST_DATA_SRC + '.out.txt', (err, data) => {
		expect(err).to.not.exist;
		expect(data).to.be.an('Object');
		expect(data).to.have.property('Bucket', 'apination-cn-data');
		expect(data).to.have.property('Key', 'staging/cn-example/transactions.json.out.txt');
	}));

createArrayWriteStream()

writes json array to S3.

utils.createReadArrayStream(TEST_DATA_SRC)
	.pipe(utils.createWriteArrayStream(TEST_DATA_SRC + '.out.json', (err, data) => {
		expect(err).to.not.exist;
		expect(data).to.be.an('Object');
		expect(data).to.have.property('Bucket', 'apination-cn-data');
		expect(data).to.have.property('Key', 'staging/cn-example/transactions.json.out.json');
	}));

loadJson()

loads JSON object from S3.

utils.loadJson(TEST_DATA_SRC).then(json => {
	expect(json).to.be.an('Array').that.has.length(2);
});

loadRemoteResources()

loads JSON objects from S3, when defined as { $src: "" }.

const input = {
	remoteResource: { $src: TEST_DATA_SRC },
	anotherResource: { foo: 'bar' }
};
utils.loadRemoteResources(input, ['remoteResource', 'anotherResource']).then(obj => {
	expect(obj).to.have.property('remoteResource').that.is.an('Array').that.has.length(2);
	expect(obj).to.have.deep.property('anotherResource.foo', 'bar');
});

Dependencies

package.json (installed automatically with npm i)

• JSONStream
• debug
• development
  • aws-sdk
  • mocha
  • chai