Merge pull request #400 from ipfs/feat/go-libp2p-0.16-and-sharding-co…

…nfig-option

feat: go-libp2p 0.16 and autosharding

.github/workflows/test.yml

@@ -7,8 +7,8 @@ on:
 # To run CI against unrelased go-ipfs or js-ipfs-* code (eg. wip PR),
 # uncomment env below and define git revisions in ./scripts/custom-runtime.sh
 
-#env:
- #IPFS_GO_EXEC: /tmp/go-ipfs/cmd/ipfs/ipfs
+env:
+ IPFS_GO_EXEC: /tmp/go-ipfs/cmd/ipfs/ipfs
  #IPFS_JS_EXEC: /tmp/js-ipfs/packages/ipfs/src/cli.js
  #IPFS_JS_MODULE: /tmp/js-ipfs/packages/ipfs/dist/cjs/src/index.js
  #IPFS_JS_HTTP_MODULE: /tmp/js-ipfs/packages/ipfs-http-client/dist/cjs/src/index.js
@@ -31,6 +31,7 @@ jobs:
  with:
  path: |
  /tmp/*-ipfs/**
+ ./go-libp2p-relay-daemon
  ~/.npm
  ./node_modules
  key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
@@ -56,6 +57,7 @@ jobs:
  with:
  path: |
  /tmp/*-ipfs/**
+ ./go-libp2p-relay-daemon
  ~/.npm
  ./node_modules
  key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
@@ -84,6 +86,7 @@ jobs:
  with:
  path: |
  /tmp/*-ipfs/**
+ ./go-libp2p-relay-daemon
  ~/.npm
  ./node_modules
  key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
@@ -106,6 +109,7 @@ jobs:
  type:
  - browser
  - webworker
+ fail-fast: false
  steps:
  - uses: actions/checkout@v2
  - uses: actions/setup-node@v2
@@ -118,6 +122,7 @@ jobs:
  with:
  path: |
  /tmp/*-ipfs/**
+ ./go-libp2p-relay-daemon
  ~/.npm
  ./node_modules
  key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
@@ -137,6 +142,7 @@ jobs:
  type:
  - electron-main
  - electron-renderer
+ fail-fast: false
  steps:
  - uses: actions/checkout@v2
  - uses: actions/setup-node@v2
@@ -149,6 +155,7 @@ jobs:
  with:
  path: |
  /tmp/*-ipfs/**
+ ./go-libp2p-relay-daemon
  ~/.npm
  ./node_modules
  key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}

.gitignore

@@ -37,4 +37,6 @@ dist
 test/test-data/go-ipfs-repo/LOCK
 test/test-data/go-ipfs-repo/LOG
 test/test-data/go-ipfs-repo/LOG.old
-types
+types
+go-libp2p-relay-daemon
+*.identity

package.json

@@ -23,6 +23,8 @@
  "npm": ">6.0.0"
  },
  "scripts": {
+ "postinstall": "./scripts/setup-relayd.sh",
+ "clean": "rimraf /tmp/js-ipfs /tmp/go-ipfs ./go-libp2p-relay-daemon",
  "lint": "aegir lint",
  "build": "aegir build",
  "pretest": "aegir build --esm-tests",
@@ -45,7 +47,9 @@
  },
  "homepage": "https://github.com/ipfs/interop#readme",
  "browser": {
- "go-ipfs": false
+ "go-ipfs": false,
+ "fs": false,
+ "execa": false
  },
  "dependencies": {
  "aegir": "^35.0.2",
@@ -55,8 +59,8 @@
  "delay": "^5.0.0",
  "detect-node": "^2.0.4",
  "ipfs-unixfs": "^6.0.3",
- "ipfsd-ctl": "^10.0.3",
  "ipfs-utils": "^9.0.1",
+ "ipfsd-ctl": "^10.0.3",
  "ipns": "^0.15.0",
  "is-ci": "^3.0.0",
  "is-os": "^1.0.1",
@@ -75,7 +79,7 @@
  "promisify-es6": "^1.0.3",
  "random-fs": "^1.0.3",
  "readable-stream-buffer-stream": "^1.0.0",
- "rimraf": "^3.0.0",
+ "rimraf": "^3.0.2",
  "uint8arrays": "^3.0.0",
  "wherearewe": "^1.0.0"
  },
@@ -97,6 +101,7 @@
  "Richard Littauer <richard.littauer@gmail.com>"
  ],
  "devDependencies": {
+ "execa": "^5.1.1",
  "go-ipfs": "^0.9.1",
  "ipfs": "^0.59.0",
  "ipfs-http-client": "^53.0.0"

scripts/custom-runtime.sh

@@ -25,7 +25,7 @@ if [ ! -d /tmp/go-ipfs ]; then
  git clone https://github.com/ipfs/go-ipfs.git
  cd go-ipfs
  # set implementation to specific commit
- git checkout CHANGEME_GO
+ git checkout 8c902b156b1c35d17ce45de2525c80bf71c77b5b # https://github.com/ipfs/go-ipfs/pull/8563
  make build
 fi
 fi

scripts/relayd_v1.config.json

@@ -0,0 +1,15 @@
+{
+ "RelayV1": {
+ "Enabled": true
+ },
+ "RelayV2": {
+ "Enabled": false
+ },
+ "Network": {
+ "ListenAddrs": ["/ip4/127.0.0.1/tcp/14111/ws"],
+ "AnnounceAddrs": ["/ip4/127.0.0.1/tcp/14111/ws"]
+ },
+ "Daemon": {
+ "PprofPort": -1
+ }
+}

scripts/relayd_v2.config.json

@@ -0,0 +1,15 @@
+{
+ "RelayV1": {
+ "Enabled": false
+ },
+ "RelayV2": {
+ "Enabled": true
+ },
+ "Network": {
+ "ListenAddrs": ["/ip4/127.0.0.1/tcp/24222/ws"],
+ "AnnounceAddrs": ["/ip4/127.0.0.1/tcp/24222/ws"]
+ },
+ "Daemon": {
+ "PprofPort": -1
+ }
+}

scripts/setup-relayd.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# This script ensures go-libp2p-relay-daemon is available
+# for use in circuit v1 and v2 tests.
+
+set -eo pipefail
+
+if ! test -e ./go-libp2p-relay-daemon/relayd; then
+ echo "Building ./go-libp2p-relay-daemon/relayd binary.."
+ rm -rf ./go-libp2p-relay-daemon
+ git clone https://github.com/libp2p/go-libp2p-relay-daemon.git
+ cd go-libp2p-relay-daemon
+ # no releases atm, so we pin implementation to specific commit
+ git checkout 65211a0b6d881086feb7c386d780f55c37dff101 # 2021-11-19
+ go build ./...
+ echo "./go-libp2p-relay-daemon/relayd is ready"
+fi

test/circuit.js

@@ -1,10 +1,13 @@
 /* eslint max-nested-callbacks: ["error", 8] */
 /* eslint-env mocha */
 
-import all from './circuit/all.js'
-import browser from './circuit/browser.js'
+import allV1 from './circuit/v1/all.js'
+import allV2 from './circuit/v2/all.js'
+import browserV1 from './circuit/v1/browser.js'
+import browserV2 from './circuit/v2/browser.js'
 import isNode from 'detect-node'
 import { connect, send, clean } from './utils/circuit.js'
+import { closeRelays } from './utils/relayd.js'
 import { daemonFactory } from './utils/daemon-factory.js'
 
 const timeout = 80 * 1000
@@ -15,40 +18,84 @@ const baseTest = {
 }
 
 describe('circuit', () => {
- let factory
+ after(closeRelays)
 
- before(async () => {
-  factory = await daemonFactory()
- })
+ // Legacy v1 (unlimited relay)
+ describe('v1', () => {
+  let factory
 
- const tests = isNode ? all : browser
+ before(async () => {
+ factory = await daemonFactory()
+ })
 
- Object.keys(tests).forEach((test) => {
- let nodeA
- let relay
- let nodeB
+ const tests = isNode ? allV1 : browserV1
 
- tests[test] = Object.assign({}, baseTest, tests[test])
+ Object.keys(tests).forEach((test) => {
+ let nodeA
+ let relay
+ let nodeB
 
- const dsc = tests[test].skip && tests[test].skip()
- ? describe.skip
- : describe
+ tests[test] = Object.assign({}, baseTest, tests[test])
 
- dsc(test, function () {
- this.timeout(tests[test].timeout)
+ const dsc = tests[test].skip && tests[test].skip()
+ ? describe.skip
+ : describe
 
- before(async () => {
- [nodeA, relay, nodeB] = await tests[test].create(factory)
- })
+ dsc(test, function () {
+ this.timeout(tests[test].timeout)
+
+ before(async () => {
+ [nodeA, relay, nodeB] = await tests[test].create(factory)
+ })
+
+ after(() => clean(factory))
 
- after(() => clean(factory))
+ it('connect', () => {
+ return tests[test].connect(nodeA, nodeB, relay)
+ })
 
- it('connect', () => {
- return tests[test].connect(nodeA, nodeB, relay)
+ it('send', () => {
+ return tests[test].send(nodeA, nodeB)
+ })
  })
+ })
+ })
+
+ // Modern v2 (limited relay)
+ // https://github.com/libp2p/specs/blob/master/relay/circuit-v2.md
+ describe('v2', () => {
+ let factory
+
+ before(async () => {
+ factory = await daemonFactory()
+ })
+
+ const tests = isNode ? allV2 : browserV2
+
+ Object.keys(tests).forEach((test) => {
+ let nodeA
+ let relay
+ let nodeB
+
+ tests[test] = Object.assign({}, baseTest, tests[test])
+
+ const dsc = tests[test].skip && tests[test].skip()
+ ? describe.skip
+ : describe
+
+ dsc(test, function () {
+ this.timeout(tests[test].timeout)
+
+ before(async () => {
+ [nodeA, relay, nodeB] = await tests[test].create(factory)
+ })
+
+ after(() => clean(factory))
 
- it('send', () => {
- return tests[test].send(nodeA, nodeB)
+ it('connect', () => {
+ return tests[test].connect(nodeA, nodeB, relay)
+ })
+ // Note: v2 provides a limited relay for things like hole punching – no send test
  })
  })
  })