Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 56ddfaa

Browse files
committed
feat: add grpc server and client
Adds a server running a gRPC endpoint over websockets, a client to access the server and a `ipfs-client` module that uses the gRPC client with HTTP fallback. So far only supports `ipfs.addAll` but the idea is to implement all streaming methods over websockets instead of HTTP, to give us bidirectional streaming and errors that work in the browser. Fixes: Depends on: - [ ] ipfs/js-ipfsd-ctl#561
1 parent 6b207d7 commit 56ddfaa

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+2952
-15
lines changed

.travis.yml

+35
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,41 @@ jobs:
164164
script:
165165
- npm run test:interface:core -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000
166166

167+
- stage: test
168+
name: js-ipfs interface tests - ipfs-client - node
169+
script:
170+
- npm run test:interface:client -- $RUN_SINCE -- -- --bail -t node
171+
172+
- stage: test
173+
name: js-ipfs interface tests - ipfs-client - chrome
174+
script:
175+
- npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser
176+
177+
- stage: test
178+
name: js-ipfs interface tests - ipfs-client - chrome webworker
179+
script:
180+
- npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --timeout 60000
181+
182+
- stage: test
183+
name: js-ipfs interface tests - ipfs-client - firefox
184+
script:
185+
- npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser --browsers FirefoxHeadless
186+
187+
- stage: test
188+
name: js-ipfs interface tests - ipfs-client - firefox webworker
189+
script:
190+
- npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --browsers FirefoxHeadless --timeout 60000
191+
192+
- stage: test
193+
name: js-ipfs interface tests - ipfs-client - electron main
194+
script:
195+
- npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-main --timeout 60000
196+
197+
- stage: test
198+
name: js-ipfs interface tests - ipfs-client - electron renderer
199+
script:
200+
- npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000
201+
167202
- stage: test
168203
name: http-api-client interface tests vs go-ipfs - node
169204
script:

examples/browser-ipns-publish/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
"devDependencies": {
2828
"delay": "^4.4.0",
2929
"execa": "^4.0.3",
30-
"ipfsd-ctl": "^7.1.1",
30+
"ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
3131
"go-ipfs": "^0.7.0",
3232
"parcel-bundler": "^1.12.4",
3333
"path": "^0.12.7",

examples/explore-ethereum-blockchain/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"devDependencies": {
1313
"ipfs": "^0.52.2",
1414
"ipfs-http-client": "^48.1.2",
15-
"ipfsd-ctl": "^7.1.1",
15+
"ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
1616
"ipld-ethereum": "^5.0.1",
1717
"test-ipfs-example": "^2.0.3"
1818
}

examples/http-client-browser-pubsub/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
"execa": "^4.0.3",
2222
"go-ipfs": "^0.7.0",
2323
"ipfs": "^0.52.2",
24-
"ipfsd-ctl": "^7.1.1",
24+
"ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
2525
"parcel-bundler": "^1.12.4",
2626
"test-ipfs-example": "^2.0.3"
2727
}

examples/http-client-bundle-webpack/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
"copy-webpack-plugin": "^5.0.4",
2626
"execa": "^4.0.3",
2727
"ipfs": "^0.52.2",
28-
"ipfsd-ctl": "^7.1.1",
28+
"ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
2929
"react-hot-loader": "^4.12.21",
3030
"rimraf": "^3.0.2",
3131
"test-ipfs-example": "^2.0.3",

examples/http-client-name-api/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"devDependencies": {
1919
"execa": "^4.0.3",
2020
"go-ipfs": "^0.7.0",
21-
"ipfsd-ctl": "^7.1.1",
21+
"ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
2222
"parcel-bundler": "^1.12.4",
2323
"rimraf": "^3.0.2",
2424
"test-ipfs-example": "^2.0.3"
+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# JS IPFS API - Example Browser - Name
2+
3+
## Setup
4+
5+
```sh
6+
npm install -g ipfs
7+
jsipfs init
8+
# Configure CORS to allow ipfs-http-client to access this IPFS node
9+
jsipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["http://127.0.0.1:8888"]'
10+
# Start the IPFS node
11+
jsipfs daemon
12+
```
13+
14+
Then in this folder run
15+
16+
```bash
17+
> npm install
18+
> npm start
19+
```
20+
21+
and open your browser at `http://127.0.0.1:8888`.
+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<!doctype html>
2+
<html>
3+
<head>
4+
<meta charset="utf-8" />
5+
<title>JS IPFS Client example</title>
6+
<style>
7+
.hidden {
8+
opacity: 0;
9+
}
10+
11+
form {
12+
padding-bottom: 1em;
13+
}
14+
</style>
15+
</head>
16+
17+
<body>
18+
<h1>ipfs-client</h1>
19+
<form id="connect-to-api">
20+
<h3>Enter IPFS API details</h3>
21+
<label for="grpc-input">
22+
GRPC:
23+
<input id="grpc-input" name="grpc-input" type="text" value="/ip4/127.0.0.1/tcp/5003" required>
24+
</label>
25+
<label for="http-input">
26+
HTTP:
27+
<input id="http-input" name="text" type="text" value="/ip4/127.0.0.1/tcp/5001" required>
28+
</label>
29+
<button id="connect-submit" type="submit">Connect</button>
30+
</form>
31+
<div id="output">
32+
</div>
33+
34+
<script src="index.js"></script>
35+
</body>
36+
</html>
+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/* eslint-disable no-console */
2+
'use strict'
3+
4+
const ipfsClient = require('ipfs-client')
5+
let ipfs
6+
7+
const COLORS = {
8+
active: 'blue',
9+
success: 'green',
10+
error: 'red'
11+
}
12+
13+
const showStatus = (text, bg) => {
14+
console.info(text)
15+
16+
const log = document.getElementById('output')
17+
18+
if (!log) {
19+
return
20+
}
21+
22+
const line = document.createElement('p')
23+
line.innerText = text
24+
line.style.color = bg
25+
26+
log.appendChild(line)
27+
}
28+
29+
async function * streamFiles () {
30+
for (let i = 0; i < 100; i++) {
31+
await new Promise((resolve) => {
32+
setTimeout(() => resolve(), 100)
33+
})
34+
35+
showStatus(`Sending /file-${i}.txt`, COLORS.active)
36+
37+
yield {
38+
path: `/file-${i}.txt`,
39+
content: `file ${i}`
40+
}
41+
}
42+
}
43+
44+
async function main (grpcApi, httpApi) {
45+
showStatus(`Connecting to ${grpcApi} using ${httpApi} as fallback`, COLORS.active)
46+
47+
ipfs = ipfsClient({
48+
grpc: grpcApi,
49+
http: httpApi
50+
})
51+
52+
const id = await ipfs.id()
53+
showStatus(`Daemon active\nID: ${id.id}`, COLORS.success)
54+
55+
for await (const file of ipfs.addAll(streamFiles(), {
56+
wrapWithDirectory: true,
57+
// this is just to show the interleaving of uploads and progress events
58+
// otherwise we'd have to upload 50 files before we see any response from
59+
// the server. do not specify this so low in production as you'll have
60+
// greatly degraded import performance
61+
fileImportConcurrency: 1,
62+
progress: (bytes, file) => {
63+
showStatus(`File progress ${file} ${bytes}`, COLORS.active)
64+
}
65+
})) {
66+
showStatus(`Added file: ${file.path} ${file.cid}`, COLORS.success)
67+
}
68+
69+
showStatus('Finished!', COLORS.success)
70+
}
71+
72+
// Event listeners
73+
document.getElementById('connect-submit').onclick = (e) => {
74+
e.preventDefault()
75+
76+
main(document.getElementById('grpc-input').value, document.getElementById('http-input').value)
77+
.catch(err => {
78+
showStatus(err.message, COLORS.error)
79+
console.error(err)
80+
})
81+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"name": "example-ipfs-client-add-files",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "index.js",
6+
"private": true,
7+
"scripts": {
8+
"clean": "rimraf ./dist",
9+
"build": "parcel build index.html --public-url '.'",
10+
"start": "parcel index.html -p 8888",
11+
"test": "test-ipfs-example"
12+
},
13+
"dependencies": {
14+
"ipfs-client": "^0.1.0"
15+
},
16+
"devDependencies": {
17+
"execa": "^4.0.3",
18+
"ipfs": "^0.52.0",
19+
"ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
20+
"parcel-bundler": "^1.12.4",
21+
"rimraf": "^3.0.2",
22+
"test-ipfs-example": "^2.0.3"
23+
},
24+
"browserslist": [
25+
"last 2 versions and not dead and > 2%"
26+
]
27+
}
+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
'use strict'
2+
3+
const path = require('path')
4+
const execa = require('execa')
5+
const { createFactory } = require('ipfsd-ctl')
6+
const df = createFactory({
7+
ipfsClientModule: require('ipfs-client'),
8+
ipfsBin: require.resolve('ipfs/src/cli.js')
9+
})
10+
const {
11+
startServer
12+
} = require('test-ipfs-example/utils')
13+
const pkg = require('./package.json')
14+
15+
async function testUI (url, http, grpc, id) {
16+
const proc = execa(require.resolve('test-ipfs-example/node_modules/.bin/nightwatch'), ['--config', require.resolve('test-ipfs-example/nightwatch.conf.js'), path.join(__dirname, 'test.js')], {
17+
cwd: path.resolve(__dirname, '../'),
18+
env: {
19+
...process.env,
20+
CI: true,
21+
IPFS_EXAMPLE_TEST_URL: url,
22+
IPFS_GRPC_API_MULTIADDR: grpc,
23+
IPFS_HTTP_API_MULTIADDR: http
24+
},
25+
all: true
26+
})
27+
proc.all.on('data', (data) => {
28+
process.stdout.write(data)
29+
})
30+
31+
await proc
32+
}
33+
34+
async function runTest () {
35+
const app = await startServer(__dirname)
36+
const daemon = await df.spawn({
37+
type: 'js',
38+
test: true,
39+
ipfsOptions: {
40+
config: {
41+
Addresses: {
42+
API: '/ip4/127.0.0.1/tcp/0',
43+
RPC: '/ip4/127.0.0.1/tcp/0'
44+
},
45+
API: {
46+
HTTPHeaders: {
47+
'Access-Control-Allow-Origin': [
48+
app.url
49+
]
50+
}
51+
}
52+
}
53+
}
54+
})
55+
56+
try {
57+
await testUI(app.url, daemon.apiAddr, daemon.grpcAddr, daemon.api.peerId.id)
58+
} finally {
59+
await daemon.stop()
60+
await app.stop()
61+
}
62+
}
63+
64+
module.exports = runTest
65+
66+
module.exports[pkg.name] = function (browser) {
67+
browser
68+
.url(process.env.IPFS_EXAMPLE_TEST_URL)
69+
.waitForElementVisible('#grpc-input')
70+
.clearValue('#grpc-input')
71+
.setValue('#grpc-input', process.env.IPFS_GRPC_API_MULTIADDR)
72+
.pause(1000)
73+
.waitForElementVisible('#http-input')
74+
.clearValue('#http-input')
75+
.setValue('#http-input', process.env.IPFS_HTTP_API_MULTIADDR)
76+
.pause(1000)
77+
.click('#connect-submit')
78+
79+
browser.expect.element('#output').text.to.contain('Added file: file-0.txt QmUDLiEJwL3vUhhXNXDF2RrCnVkSB2LemWYffpCCPcQCeU')
80+
81+
browser.end()
82+
}

package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"version": "1.0.0",
44
"description": "JavaScript implementation of the IPFS specification",
55
"scripts": {
6-
"postinstall": "lerna bootstrap",
6+
"postinstall": "lerna bootstrap && npm run build -- --scope=ipfs-grpc-protocol",
77
"link": "lerna link",
88
"reset": "lerna run clean && rimraf packages/*/node_modules node_modules",
99
"test": "lerna run test",
@@ -16,6 +16,7 @@
1616
"test:external": "lerna run test:external",
1717
"test:cli": "lerna run test:cli",
1818
"test:interop": "lerna run test:interop",
19+
"test:interface:client": "lerna run test:interface:client",
1920
"test:interface:core": "lerna run test:interface:core",
2021
"test:interface:http-go": "lerna run test:interface:http-go",
2122
"test:interface:http-js": "lerna run test:interface:http-js",

packages/interface-ipfs-core/src/add-all.js

+16
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,22 @@ module.exports = (common, options) => {
422422
expect(files[0].size).to.equal(18)
423423
})
424424

425+
it('should add directories with metadata', async () => {
426+
const files = await all(ipfs.addAll([{
427+
path: '/foo',
428+
mode: 0o123,
429+
mtime: {
430+
secs: 1000,
431+
nsecs: 0
432+
}
433+
}]))
434+
435+
expect(files.length).to.equal(1)
436+
expect(files[0].cid.toString()).to.equal('QmaZTosBmPwo9LQ48ESPCEcNuX2kFxkpXYy8i3rxqBdzRG')
437+
expect(files[0].cid.codec).to.equal('dag-pb')
438+
expect(files[0].size).to.equal(11)
439+
})
440+
425441
it('should support bidirectional streaming', async function () {
426442
let progressInvoked
427443

packages/ipfs-cli/src/commands/daemon.js

+5-3
Original file line numberDiff line numberDiff line change
@@ -83,15 +83,17 @@ module.exports = {
8383

8484
try {
8585
await daemon.start()
86-
// @ts-ignore - _httpApi is possibly undefined
86+
// @ts-ignore - _apiServers is possibly undefined
8787
daemon._httpApi._apiServers.forEach(apiServer => {
88-
print(`API listening on ${apiServer.info.ma}`)
88+
print(`HTTP API listening on ${apiServer.info.ma}`)
8989
})
90+
// @ts-ignore - _grpcServer is possibly undefined
91+
print(`gRPC listening on ${daemon._grpcServer.multiaddr}`)
9092
// @ts-ignore - _httpGateway is possibly undefined
9193
daemon._httpGateway._gatewayServers.forEach(gatewayServer => {
9294
print(`Gateway (read only) listening on ${gatewayServer.info.ma}`)
9395
})
94-
// @ts-ignore - _httpApi is possibly undefined
96+
// @ts-ignore - _apiServers is possibly undefined
9597
daemon._httpApi._apiServers.forEach(apiServer => {
9698
print(`Web UI available at ${toUri(apiServer.info.ma)}/webui`)
9799
})

0 commit comments

Comments
 (0)