diff --git a/QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe b/QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe new file mode 100644 index 0000000000..21f06bb5ab --- /dev/null +++ b/QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe @@ -0,0 +1 @@ +{ "Data": "another", "Links": [ { "Name": "some link", "Hash": "QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V", "Size": 8 } ] } diff --git a/package.json b/package.json index 98114eaaee..e156c32087 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "lodash.get": "^4.2.1", "lodash.set": "^4.1.0", "multiaddr": "^1.4.1", + "ndjson": "^1.4.3", "path-exists": "^3.0.0", "peer-book": "0.1.0", "peer-id": "^0.6.6", diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 080ed22be3..50ce5995d8 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -35,6 +35,49 @@ function checkPath (inPath, recursive) { return inPath } +function daemonOn (res, inPath, ipfs) { + const files = [] + if (res.length !== 0) { + const index = inPath.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + if (fs.statSync(element).isDirectory()) { + callback() + } else { + const filePair = { + path: element.substring(index + 1, element.length), + content: fs.createReadStream(element) + } + files.push(filePair) + callback() + } + }, (err) => { + if (err) { + throw err + } + ipfs.add(files, (err, res) => { + if (err) { + throw err + } + res.forEach((goRes) => { + console.log('added', goRes.Hash, goRes.Name) + }) + }) + }) + } else { + const filePair = { + path: inPath.substring(inPath.lastIndexOf('/') + 1, inPath.length), + content: fs.createReadStream(inPath) + } + files.push(filePair) + ipfs.add(files, (err, res) => { + if (err) { + throw err + } + console.log('added', res[0].Hash, res[0].Name) + }) + } + return +} module.exports = Command.extend({ desc: 'Add a file to IPFS using the UnixFS data format', @@ -59,36 +102,40 @@ module.exports = Command.extend({ if (err) { throw err } - const i = ipfs.files.add() - var filePair - i.on('data', (file) => { - console.log('added', bs58.encode(file.multihash).toString(), file.path) - }) - i.once('end', () => { - return - }) - if (res.length !== 0) { - const index = inPath.lastIndexOf('/') - async.eachLimit(res, 10, (element, callback) => { - if (!fs.statSync(element).isDirectory()) { - i.write({ - path: element.substring(index + 1, element.length), - stream: fs.createReadStream(element) - }) - } - callback() - }, (err) => { - if (err) { - throw err - } - i.end() - }) + if (utils.isDaemonOn()) { + daemonOn(res, inPath, ipfs) } else { - rs = fs.createReadStream(inPath) - inPath = inPath.substring(inPath.lastIndexOf('/') + 1, inPath.length) - filePair = {path: inPath, stream: rs} - i.write(filePair) - i.end() + const i = ipfs.files.add() + var filePair + i.on('data', (file) => { + console.log('added', bs58.encode(file.multihash).toString(), file.path) + }) + i.once('end', () => { + return + }) + if (res.length !== 0) { + const index = inPath.lastIndexOf('/') + async.eachLimit(res, 10, (element, callback) => { + if (!fs.statSync(element).isDirectory()) { + i.write({ + path: element.substring(index + 1, element.length), + stream: fs.createReadStream(element) + }) + } + callback() + }, (err) => { + if (err) { + throw err + } + i.end() + }) + } else { + rs = fs.createReadStream(inPath) + inPath = inPath.substring(inPath.lastIndexOf('/') + 1, inPath.length) + filePair = {path: inPath, stream: rs} + i.write(filePair) + i.end() + } } }) }) diff --git a/src/cli/commands/files/cat.js b/src/cli/commands/files/cat.js index fe61d61316..7ff38177cb 100644 --- a/src/cli/commands/files/cat.js +++ b/src/cli/commands/files/cat.js @@ -22,6 +22,15 @@ module.exports = Command.extend({ if (err) { throw err } + if (utils.isDaemonOn()) { + ipfs.cat(path, (err, res) => { + if (err) { + throw err + } + console.log(res.toString()) + }) + return + } ipfs.files.cat(path, (err, res) => { if (err) { throw (err) diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js new file mode 100644 index 0000000000..9f9441e78f --- /dev/null +++ b/src/http-api/resources/files.js @@ -0,0 +1,159 @@ +'use strict' + +const bs58 = require('bs58') +// const ndjson = require('ndjson') +// const async = require('async') +const Readable = require('stream').Readable +const multipart = require('ipfs-multipart') +const debug = require('debug') +const log = debug('http-api:files') +log.error = debug('http-api:files:error') + +exports = module.exports + +// common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args` +exports.parseKey = (request, reply) => { + if (!request.query.arg) { + return reply("Argument 'key' is required").code(400).takeover() + } + + try { + return reply({ + key: new Buffer(bs58.decode(request.query.arg)) + }) + } catch (err) { + log.error(err) + return reply({ + Message: 'invalid ipfs ref path', + Code: 0 + }).code(500).takeover() + } +} + +exports.add = { + // pre request handler that parses the args and returns `node` which is assigned to `request.pre.args` + handler: (request, reply) => { + if (!request.payload) { + return reply('Array, Buffer, or String is required').code(400).takeover() + } + const parser = multipart.reqParser(request.payload) + var file = false + var filePair + const resArr = [] + // let serialArr + // console.log(serialArr) + var i = request.server.app.ipfs.files.add() + // var serialize = ndjson.stringify() + + i.on('data', (file) => { + resArr.push({ + Name: file.path, + Hash: bs58.encode(file.multihash).toString() + }) + }) + + i.on('end', () => { + if (resArr.length === 0 && file) { + return reply({ + Message: 'Failed to add files', + Code: 0 + }).code(500) + } + + /* serialize.on('data', (line) => { + var serialArr = line + console.log(line) + return reply(serialArr) + //console.log(line) + }) + + async.eachSeries(resArr, (item, callback) => { + serialize.write(item) + callback() + }, (done) => { + serialize.end() + }) + + serialize.on('end', () => { + //console.log(serialArr.length) + //return reply(serialArr) + }) */ + return reply(resArr) + }) + + parser.on('file', (fileName, fileStream) => { + var rs = new Readable() + var init = false + rs._read = () => { + if (init) { + return + } + init = true + } + fileStream.on('data', (data) => { + rs.push(data) + file = true + }) + fileStream.on('end', () => { + rs.push(null) + filePair = { + path: fileName, + stream: rs + } + i.write(filePair) + }) + }) + + parser.on('end', () => { + if (!file) { + return reply("File argument 'data' is required").code(400).takeover() + } + i.end() + }) + } +} + +exports.cat = { + // uses common parseKey method that returns a `key` + parseArgs: exports.parseKey, + + // main route handler which is called after the above `parseArgs`, but only if the args were valid + handler: (request, reply) => { + const key = request.pre.args.key + + request.server.app.ipfs.files.cat(key, (err, ee) => { + if (err) { + log.error(err) + return reply({ + Message: 'Failed to cat file: ' + err, + Code: 0 + }).code(500) + } + ee.on('file', (data) => { + return reply(data.stream) + }) + }) + } +} + +exports.get = { + // uses common parseKey method that returns a `key` + parseArgs: exports.parseKey, + + // main route handler which is called after the above `parseArgs`, but only if the args were valid + handler: (request, reply) => { + const key = request.pre.args.key + + request.server.app.ipfs.files.get(key, (err, ee) => { + if (err) { + log.error(err) + return reply({ + Message: 'Failed to get object: ' + err, + Code: 0 + }).code(500) + } + + return reply(ee) + }) + } +} diff --git a/src/http-api/resources/index.js b/src/http-api/resources/index.js index 036e09b212..da244797eb 100644 --- a/src/http-api/resources/index.js +++ b/src/http-api/resources/index.js @@ -8,3 +8,4 @@ exports.object = require('./object') exports.config = require('./config') exports.block = require('./block') exports.swarm = require('./swarm') +exports.files = require('./files') diff --git a/src/http-api/routes/files.js b/src/http-api/routes/files.js new file mode 100644 index 0000000000..d7f7b6381d --- /dev/null +++ b/src/http-api/routes/files.js @@ -0,0 +1,41 @@ +'use strict' + +const resources = require('./../resources') + +module.exports = (server) => { + const api = server.select('API') + + api.route({ + method: '*', + path: '/api/v0/add', + config: { + payload: { + parse: false, + output: 'stream' + }, + handler: resources.files.add.handler + } + }) + + api.route({ + method: '*', + path: '/api/v0/cat', + config: { + pre: [ + { method: resources.files.cat.parseArgs, assign: 'args' } + ], + handler: resources.files.cat.handler + } + }) + + api.route({ + method: '*', + path: '/api/v0/get', + config: { + pre: [ + { method: resources.files.get.parseArgs, assign: 'args' } + ], + handler: resources.files.get.handler + } + }) +} diff --git a/src/http-api/routes/index.js b/src/http-api/routes/index.js index de8e27bf85..fd478ce30b 100644 --- a/src/http-api/routes/index.js +++ b/src/http-api/routes/index.js @@ -8,5 +8,6 @@ module.exports = (server) => { require('./object')(server) // require('./repo')(server) require('./config')(server) + require('./files')(server) require('./swarm')(server) } diff --git a/test/cli-tests/test-files.js b/test/cli-tests/test-files.js new file mode 100644 index 0000000000..7916d01c44 --- /dev/null +++ b/test/cli-tests/test-files.js @@ -0,0 +1,96 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const nexpect = require('nexpect') +const HttpAPI = require('../../src/http-api') +const repoPath = require('./index').repoPath +const _ = require('lodash') + +describe('files', () => { + const env = _.clone(process.env) + env.IPFS_PATH = repoPath + + describe('api offline', () => { + it('add', (done) => { + nexpect.spawn('node', [process.cwd() + '/src/cli/bin.js', 'files', 'add', process.cwd() + '/test/test-data/node.json'], {env}) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]) + .to.equal('added QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe node.json') + done() + }) + }) + + it('get', (done) => { + nexpect.spawn('node', [process.cwd() + '/src/cli/bin.js', 'files', 'get', 'QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe'], {env}) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + done() + }) + }) + + it('cat', (done) => { + nexpect.spawn('node', [process.cwd() + '/src/cli/bin.js', 'files', 'cat', 'QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe'], {env}) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]) + .to.equal('{ "Data": "another", "Links": [ { "Name": "some link", "Hash": "QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V", "Size": 8 } ] }') + done() + }) + }) + }) + + describe('api running', () => { + let httpAPI + + before((done) => { + httpAPI = new HttpAPI(repoPath) + httpAPI.start((err) => { + expect(err).to.not.exist + done() + }) + }) + + after((done) => { + httpAPI.stop((err) => { + expect(err).to.not.exist + done() + }) + }) + + it('add', (done) => { + nexpect.spawn('node', [process.cwd() + '/src/cli/bin.js', 'files', 'add', process.cwd() + '/test/test-data/node.json'], {env}) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]) + .to.equal('added QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe node.json') + done() + }) + }) + + it.skip('get', (done) => { + nexpect.spawn('node', [process.cwd() + '/src/cli/bin.js', 'files', 'get', 'QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe'], {env}) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + done() + }) + }) + + it('cat', (done) => { + nexpect.spawn('node', [process.cwd() + '/src/cli/bin.js', 'files', 'cat', 'QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe'], {env}) + .run((err, stdout, exitcode) => { + expect(err).to.not.exist + expect(exitcode).to.equal(0) + expect(stdout[0]) + .to.equal('{ "Data": "another", "Links": [ { "Name": "some link", "Hash": "QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V", "Size": 8 } ] }') + done() + }) + }) + }) +}) diff --git a/test/http-api-tests/test-files.js b/test/http-api-tests/test-files.js new file mode 100644 index 0000000000..e78b16ac96 --- /dev/null +++ b/test/http-api-tests/test-files.js @@ -0,0 +1,243 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const APIctl = require('ipfs-api') +const fs = require('fs') +const FormData = require('form-data') +const streamToPromise = require('stream-to-promise') +const Readable = require('stream').Readable + +module.exports = (httpAPI) => { + describe('files', () => { + describe('api', () => { + let api + + it('api', () => { + api = httpAPI.server.select('API') + }) + + describe('/files/add', () => { + it('returns 400 if no tuple is provided', (done) => { + const form = new FormData() + const headers = form.getHeaders() + + streamToPromise(form).then((payload) => { + api.inject({ + method: 'POST', + url: '/api/v0/add', + headers: headers, + payload: payload + }, (res) => { + expect(res.statusCode).to.equal(400) + done() + }) + }) + }) + + it('adds a file', (done) => { + const form = new FormData() + const filePath = 'test/test-data/node.json' + form.append('file', fs.createReadStream(filePath)) + const headers = form.getHeaders() + + streamToPromise(form).then((payload) => { + api.inject({ + method: 'POST', + url: '/api/v0/add', + headers: headers, + payload: payload + }, (res) => { + expect(res.statusCode).to.equal(200) + done() + }) + }) + }) + + it('adds multiple files', (done) => { + const form = new FormData() + const filePath = 'test/test-data/hello' + const filePath2 = 'test/test-data/otherconfig' + form.append('file', fs.createReadStream(filePath)) + form.append('file', fs.createReadStream(filePath2)) + const headers = form.getHeaders() + + streamToPromise(form).then((payload) => { + api.inject({ + method: 'POST', + url: '/api/v0/add', + headers: headers, + payload: payload + }, (res) => { + expect(res.statusCode).to.equal(200) + done() + }) + }) + }) + }) + + describe('/files/cat', () => { + it('returns 400 for request without argument', (done) => { + api.inject({ + method: 'GET', + url: '/api/v0/cat' + }, (res) => { + expect(res.statusCode).to.equal(400) + expect(res.result).to.be.a('string') + done() + }) + }) + + it('returns 500 for request with invalid argument', (done) => { + api.inject({ + method: 'GET', + url: '/api/v0/cat?arg=invalid' + }, (res) => { + expect(res.statusCode).to.equal(500) + expect(res.result.Message).to.be.a('string') + done() + }) + }) + + it('returns a stream', (done) => { + api.inject({ + method: 'GET', + url: '/api/v0/cat?arg=QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + }, (res) => { + expect(res.statusCode).to.equal(200) + done() + }) + }) + }) + + describe('/files/get', () => { + it('returns 400 for request without argument', (done) => { + api.inject({ + method: 'GET', + url: '/api/v0/get' + }, (res) => { + expect(res.statusCode).to.equal(400) + expect(res.result).to.be.a('string') + done() + }) + }) + + it('returns 500 for request with invalid argument', (done) => { + api.inject({ + method: 'GET', + url: '/api/v0/get?arg=invalid' + }, (res) => { + expect(res.statusCode).to.equal(500) + expect(res.result.Message).to.be.a('string') + done() + }) + }) + + it('returns an event emitter', (done) => { + api.inject({ + method: 'GET', + url: '/api/v0/get?arg=QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + }, (res) => { + expect(res.statusCode).to.equal(200) + done() + }) + }) + }) + }) + + describe('using js-ipfs-api', () => { + var ctl + + it('start IPFS API ctl', (done) => { + ctl = APIctl('/ip4/127.0.0.1/tcp/6001') + done() + }) + + describe('ipfs.add', () => { + it('adds two files under a chunk Size', (done) => { + const rs = new Readable() + const rs2 = new Readable() + var files = [] + const buffered = fs.readFileSync('test/test-data/hello') + const buffered2 = fs.readFileSync('test/test-data/otherconfig') + rs.push(buffered) + rs.push(null) + rs2.push(buffered2) + rs2.push(null) + const filePair = {path: 'hello', content: rs} + const filePair2 = {path: 'otherconfig', content: rs2} + files.push(filePair) + files.push(filePair2) + + ctl.add(files, (err, res) => { + expect(err).to.not.exist + expect(res[0].Name).to.equal('hello') + expect(res[0].Hash).to.equal('QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o') + expect(res[1].Name).to.equal('otherconfig') + expect(res[1].Hash).to.equal('QmayedZNznnEbHtyfjeQvvt29opSLjYjLtLqwfwSWq28ds') + done() + }) + }) + + it('adds a large file > a chunk', (done) => { + const rs = new Readable() + var files = [] + const buffered = fs.readFileSync('test/test-data/1.2MiB.txt') + rs.push(buffered) + rs.push(null) + const filePair = {path: '1.2MiB.txt', content: rs} + files.push(filePair) + + ctl.add(filePair, (err, res) => { + expect(err).to.not.exist + expect(res[0].Name).to.equal('1.2MiB.txt') + expect(res[0].Hash).to.equal('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') + done() + }) + }) + + it('adds a buffer', (done) => { + const buffer = new Buffer('hello world') + ctl.add(buffer, (err, res) => { + expect(err).to.not.exist + expect(res[0].Hash).to.equal('Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD') + done() + }) + }) + + it('adds a url', (done) => { + const url = 'https://raw.githubusercontent.com/ipfs/js-ipfs-api/2a9cc63d7427353f2145af6b1a768a69e67c0588/README.md' + ctl.add(url, (err, res) => { + expect(err).to.not.exist + const added = res[0] != null ? res[0] : res + expect(added).to.have.a.property('Hash', 'QmZmHgEX9baxUn3qMjsEXQzG6DyNcrVnwieQQTrpDdrFvt') + done() + }) + }) + }) + + describe('ipfs.cat', () => { + it('returns error for request without argument', (done) => { + ctl.cat(null, (err, result) => { + expect(err).to.exist + done() + }) + }) + + it('returns error for request with invalid argument', (done) => { + ctl.cat('invalid', (err, result) => { + expect(err).to.exist + done() + }) + }) + + it('returns a stream', (done) => { + ctl.cat('QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o', (err, result) => { + expect(err).to.not.exist + done() + }) + }) + }) + }) + }) +} diff --git a/test/test-data/1.2MiB.txt b/test/test-data/1.2MiB.txt new file mode 100644 index 0000000000..6e306c55ab Binary files /dev/null and b/test/test-data/1.2MiB.txt differ