From bb2f8f936e813216769fcebb64b28f92036db921 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Fri, 11 Oct 2024 07:58:28 -0300 Subject: [PATCH 01/21] First commit --- .gitignore | 3 +- README.md | 47 +++-- bin.js | 18 ++ bin/client.js | 76 ------- bin/keygen.js | 87 +------- bin/login.js | 11 + bin/server.js | 174 ++-------------- constants.js | 6 - index.js | 208 +++++++++++++++++++ lib/bin/keygen.js | 42 ++++ lib/bin/login.js | 35 ++++ lib/bin/server.js | 103 ++++++++++ lib/client-socket.js | 67 ------ lib/constants.js | 4 + lib/download.js | 141 ------------- lib/file-to-keypair.js | 11 + lib/get-known-peer.js | 29 ++- lib/local-tunnel.js | 250 ----------------------- messages.js => lib/protocols/messages.js | 0 lib/{ => protocols}/shell.js | 134 ++++++++---- lib/question.js | 16 ++ lib/upload.js | 131 ------------ package.json | 45 ++-- test/basic.js | 132 ------------ test/bin.js | 61 ++++++ test/helpers/index.js | 157 +++----------- test/lib.js | 75 +++++++ 27 files changed, 803 insertions(+), 1260 deletions(-) create mode 100755 bin.js delete mode 100755 bin/client.js mode change 100755 => 100644 bin/keygen.js create mode 100644 bin/login.js mode change 100755 => 100644 bin/server.js delete mode 100644 constants.js create mode 100644 index.js create mode 100644 lib/bin/keygen.js create mode 100644 lib/bin/login.js create mode 100644 lib/bin/server.js delete mode 100644 lib/client-socket.js create mode 100644 lib/constants.js delete mode 100644 lib/download.js create mode 100644 lib/file-to-keypair.js delete mode 100644 lib/local-tunnel.js rename messages.js => lib/protocols/messages.js (100%) rename lib/{ => protocols}/shell.js (51%) create mode 100644 lib/question.js delete mode 100644 lib/upload.js delete mode 100644 test/basic.js create mode 100644 test/bin.js create mode 100644 test/lib.js diff --git a/.gitignore b/.gitignore index d5f19d8..9770d6e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ -node_modules +node_modules/ package-lock.json +coverage/ diff --git a/README.md b/README.md index 3b2da7a..1924f01 100644 --- a/README.md +++ b/README.md @@ -3,49 +3,68 @@ Spawn shells anywhere. Fully peer-to-peer, authenticated, and end to end encrypted. ## Install + ``` npm i -g hypershell ``` ## Usage + ```shell -# Create keys -hypershell-keygen [-f keyfile] [-c comment] +# Create a key +hypershell keygen [-f keyfile] [-c comment] # Create a P2P server -hypershell-server [-f keyfile] [--firewall filename] [--disable-firewall] [--protocol name] +hypershell server [-f keyfile] [--firewall filename] [--disable-firewall] [--protocol name] # Connect to a P2P shell -hypershell [-f keyfile] +hypershell login [-f keyfile] # Local tunnel that forwards to remote host -hypershell -L [address:]port:host:hostport +hypershell tunnel -L [address:]port:host:hostport # Copy files (download and upload) -hypershell-copy <[@host:]source> <[@host:]target> [-f keyfile] +hypershell copy <[@host:]source> <[@host:]target> [-f keyfile] ``` -Use `--help` with any command for more information, for example `hypershell-server --help`. +Use `--help` with any command for more information, for example `hypershell server --help`. + +It can also be imported as a library: + +```js +const Hypershell = require('hypershell') + +const keyPair = Hypershell.keygen({ filename, comment }) + +const closeServer = await Hypershell.server({ firewall }) +const closeServer = await Hypershell.login({ firewall }) + +``` ## First steps + Keys are automatically created with a default filename on first run. Otherwise, you can first do: -```bash -hypershell-keygen + +```sh +hypershell keygen ``` Just connect to servers (they have to allow your public key): -```bash -hypershell + +```sh +hypershell login ``` You could also create a server: -```bash -hypershell-server + +```sh +hypershell server ``` -`~/.hypershell/authorized_peers` file will be empty, denying all connections by default.\ +`~/.hypershell/authorized_peers` file will be empty, denying all connections by default. + Public keys can be added to the list to allow them in real-time. Or you can use the `--disable-firewall` flag to allow anyone to connect, useful for public services like game servers. diff --git a/bin.js b/bin.js new file mode 100755 index 0000000..748ef00 --- /dev/null +++ b/bin.js @@ -0,0 +1,18 @@ +#!/usr/bin/env node + +const { program } = require('commander') +const safetyCatch = require('safety-catch') +const pkg = require('./package.json') + +const main = program + .version(pkg.version) + .description(pkg.description) + .addCommand(require('./bin/keygen.js')) + .addCommand(require('./bin/server.js')) + .addCommand(require('./bin/login.js')) + +main.parseAsync().catch(err => { + safetyCatch(err) + console.error('error: ' + err.message) + process.exit(1) +}) diff --git a/bin/client.js b/bin/client.js deleted file mode 100755 index a6ba3b2..0000000 --- a/bin/client.js +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env node - -const path = require('path') -const fs = require('fs') -const { Command } = require('commander') -const Protomux = require('protomux') -const DHT = require('hyperdht') -const goodbye = require('graceful-goodbye') -const HypercoreId = require('hypercore-id-encoding') -const { SHELLDIR } = require('../constants.js') -const { ClientSocket } = require('../lib/client-socket.js') -const { ShellClient } = require('../lib/shell.js') -const { LocalTunnelClient } = require('../lib/local-tunnel.js') -const keygen = require('./keygen.js') -const getKnownPeer = require('../lib/get-known-peer.js') - -const program = new Command() - -program - .description('Connect to a P2P shell.') - .argument('', 'Public key or name of the server') - .option('-f ', 'Filename of the client seed key.', path.join(SHELLDIR, 'peer')) - .option('-L <[address:]port:host:hostport...>', 'Local port forwarding.') - // .option('--primary-key ', 'Inline primary key for the client.') - .option('--testnet', 'Use a local testnet.', false) - .action(cmd) - .parseAsync() - -async function cmd (serverPublicKey, options = {}) { - const keyfile = path.resolve(options.f) - - if (!fs.existsSync(keyfile)) { - await keygen({ f: keyfile }) - } - - if (options.L) { - // Partially hardcoded "ClientSocket" here as tunnels behaves different, until we can organize better the dht, socket, and mux objects - - serverPublicKey = getKnownPeer(serverPublicKey) - - const seed = HypercoreId.decode(fs.readFileSync(keyfile, 'utf8').trim()) - const keyPair = DHT.keyPair(seed) - - const node = new DHT({ bootstrap: options.testnet ? [{ host: '127.0.0.1', port: 40838 }] : undefined }) - goodbye(() => node.destroy(), 2) - - for (const config of options.L) { - const tunnel = new LocalTunnelClient(config, { node, keyPair, serverPublicKey }) - await tunnel.ready() - - goodbye(() => tunnel.close(), 1) - - console.log('Tunnel on TCP', getHost(tunnel.server.address().address) + ':' + tunnel.server.address().port) - } - - return - } - - if (options.R) errorAndExit('-R not supported') - - const { node, socket } = ClientSocket({ keyfile, serverPublicKey, testnet: options.testnet }) - const mux = new Protomux(socket) - - const shell = new ShellClient(this.rawArgs, { node, socket, mux }) - shell.open() -} - -function getHost (address) { - if (address === '::' || address === '0.0.0.0') return 'localhost' - return address -} - -function errorAndExit (message) { - console.error('Error:', message) - process.exit(1) -} diff --git a/bin/keygen.js b/bin/keygen.js old mode 100755 new mode 100644 index 0b47638..70a7723 --- a/bin/keygen.js +++ b/bin/keygen.js @@ -1,82 +1,7 @@ -#!/usr/bin/env node +const { createCommand } = require('commander') -const fs = require('fs') -const path = require('path') -const readline = require('readline') -const { Command } = require('commander') -const Keychain = require('keypear') -const DHT = require('hyperdht') -const HypercoreId = require('hypercore-id-encoding') -const { SHELLDIR } = require('../constants.js') - -const isModule = require.main !== module - -if (isModule) { - module.exports = cmd -} else { - const program = new Command() - - program - .description('Create keys of type ed25519 for use by hypercore-protocol.') - .option('-f ', 'Filename of the seed key file.') - .option('-c ', 'Provides a new comment.') - .action(cmd) - .parseAsync() -} - -async function cmd (options = {}) { - console.log('Generating key.') - - let keyfile - if (!options.f) { - keyfile = path.join(SHELLDIR, 'peer') - - const answer = await question('Enter file in which to save the key (' + keyfile + '): ') - const filename = answer.trim() - if (filename) { - keyfile = path.resolve(filename) - } - } else { - keyfile = path.resolve(options.f) - } - - const comment = options.c ? (' # ' + options.c) : '' - - if (fs.existsSync(keyfile)) { - if (isModule) { - console.log() - return - } - - errorAndExit(keyfile + ' already exists.') // Overwrite (y/n)? - } - - const seed = Keychain.seed() - fs.mkdirSync(path.dirname(keyfile), { recursive: true }) - fs.writeFileSync(keyfile, HypercoreId.encode(seed) + comment + '\n', { flag: 'wx', mode: '600' }) - - console.log('Your key has been saved in', keyfile) - console.log('The public key is:') - console.log(HypercoreId.encode(DHT.keyPair(seed).publicKey)) - - if (isModule) console.log() -} - -function question (query = '') { - return new Promise(resolve => { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout - }) - - rl.question(query, function (answer) { - rl.close() - resolve(answer) - }) - }) -} - -function errorAndExit (message) { - console.error('Error:', message) - process.exit(1) -} +module.exports = createCommand('keygen') + .description('create keys of type ed25519') + .option('-f ', 'filename of the seed key file') + .option('-c ', 'provides a new comment') + .action(require('../lib/bin/keygen.js')) diff --git a/bin/login.js b/bin/login.js new file mode 100644 index 0000000..0937cc3 --- /dev/null +++ b/bin/login.js @@ -0,0 +1,11 @@ +const path = require('path') +const { createCommand } = require('commander') +const constants = require('../lib/constants.js') + +module.exports = createCommand('login') + .description('connect to a P2P shell') + .argument('', 'public key or name of the server') + .option('-f ', 'filename of the client seed key', path.join(constants.dir, 'id')) + .option('-L <[address:]port:host:hostport...>', 'local port forwarding') + .option('--bootstrap ', 'custom dht nodes') + .action(require('../lib/bin/login.js')) diff --git a/bin/server.js b/bin/server.js old mode 100755 new mode 100644 index a3c80b8..1576379 --- a/bin/server.js +++ b/bin/server.js @@ -1,162 +1,12 @@ -#!/usr/bin/env node - -const fs = require('fs') -const path = require('path') -const { Command } = require('commander') -const DHT = require('hyperdht') -const goodbye = require('graceful-goodbye') -const Protomux = require('protomux') -const readFile = require('read-file-live') -const HypercoreId = require('hypercore-id-encoding') -const { SHELLDIR } = require('../constants.js') -const { waitForSocketTermination } = require('../lib/client-socket.js') -const { ShellServer } = require('../lib/shell.js') -const { UploadServer } = require('../lib/upload.js') -const { DownloadServer } = require('../lib/download.js') -const { LocalTunnelServer } = require('../lib/local-tunnel.js') -const configs = require('tiny-configs') -const keygen = require('./keygen.js') - -const PROTOCOLS = ['shell', 'upload', 'download', 'tunnel'] - -const program = new Command() - -program - .description('Create a P2P shell server.') - .option('-f ', 'Filename of the server seed key.', path.join(SHELLDIR, 'peer')) - // .option('--key ', 'Inline key for the server.') - .option('--firewall ', 'List of allowed public keys.', path.join(SHELLDIR, 'authorized_peers')) - .option('--disable-firewall', 'Allow anyone to connect.', false) - .option('--protocol ', 'List of allowed protocols.') - .option('--tunnel-host ', 'Restrict tunneling to a limited set of hosts.') - .option('--tunnel-port ', 'Restrict tunneling to a limited set of ports.') - .option('--testnet', 'Use a local testnet.', false) - .action(cmd) - .parseAsync() - -async function cmd (options = {}) { - const keyfile = path.resolve(options.f) - const firewall = path.resolve(options.firewall) - const protocols = options.protocol || PROTOCOLS - - if (!fs.existsSync(keyfile)) { - await keygen({ f: keyfile }) - } - - let allowed = options.disableFirewall === true - if (!allowed) { - allowed = readAuthorizedPeers(firewall) - const unwatchFirewall = readFile(firewall, function (buf) { - allowed = readAuthorizedPeers(buf) - }) - goodbye(() => unwatchFirewall(), 3) - } - const seed = HypercoreId.decode(fs.readFileSync(keyfile, 'utf8').trim()) - const keyPair = DHT.keyPair(seed) - - const node = new DHT({ bootstrap: options.testnet ? [{ host: '127.0.0.1', port: 40838 }] : undefined }) - goodbye(() => node.destroy(), 3) - - const server = node.createServer({ firewall: onFirewall }) - goodbye(() => server.close(), 2) - - server.on('connection', onconnection.bind(server, { protocols, options })) - - await server.listen(keyPair) - - if (protocols === PROTOCOLS) { - console.log('To connect to this shell, on another computer run:') - console.log('hypershell ' + HypercoreId.encode(keyPair.publicKey)) - } else { - console.log('Running server with restricted protocols') - console.log('Server key: ' + HypercoreId.encode(keyPair.publicKey)) - } - console.log() - - function onFirewall (remotePublicKey, remoteHandshakePayload) { - if (allowed === true) { - console.log('Firewall allowed:', HypercoreId.encode(remotePublicKey)) - return false - } - - for (const publicKey of allowed) { - if (remotePublicKey.equals(publicKey)) { - console.log('Firewall allowed:', HypercoreId.encode(remotePublicKey)) - return false - } - } - - console.log('Firewall denied:', HypercoreId.encode(remotePublicKey)) - return true - } -} - -function onconnection ({ protocols, options }, socket) { - const node = this.dht - - socket.on('end', () => socket.end()) - socket.on('close', () => console.log('Connection closed', HypercoreId.encode(socket.remotePublicKey))) - socket.on('error', function (error) { - if (error.code === 'ECONNRESET' || error.code === 'ETIMEDOUT') return - console.error(error.code, error) - }) - - socket.setKeepAlive(5000) - - const unregisterSocket = goodbye(() => { - socket.end() - return waitForSocketTermination(socket) - }, 1) - socket.once('close', () => unregisterSocket()) - - const mux = new Protomux(socket) - - if (protocols.includes('shell')) { - mux.pair({ protocol: 'hypershell' }, function () { - const shell = new ShellServer({ node, socket, mux }) - if (!shell.channel) return - shell.open() - }) - } - - if (protocols.includes('upload')) { - mux.pair({ protocol: 'hypershell-upload' }, function () { - const upload = new UploadServer({ node, socket, mux }) - if (!upload.channel) return - upload.open() - }) - } - - if (protocols.includes('download')) { - mux.pair({ protocol: 'hypershell-download' }, function () { - const download = new DownloadServer({ node, socket, mux }) - if (!download.channel) return - download.open() - }) - } - - if (protocols.includes('tunnel')) { - mux.pair({ protocol: 'hypershell-tunnel-local' }, function () { - const tunnel = new LocalTunnelServer({ node, socket, mux, options }) - if (!tunnel.channel) return - tunnel.open() - }) - } -} - -function readAuthorizedPeers (filename) { - if (typeof filename === 'string' && !fs.existsSync(filename)) { - console.log('Notice: creating default firewall', filename) - fs.mkdirSync(path.dirname(filename), { recursive: true }) - fs.writeFileSync(filename, '# \n', { flag: 'wx' }) - } - - try { - const list = typeof filename === 'string' ? fs.readFileSync(filename, 'utf8') : filename - return configs.parse(list) - .map(v => HypercoreId.decode(v)) - } catch (error) { - if (error.code === 'ENOENT') return [] - throw error - } -} +const { createCommand } = require('commander') + +module.exports = createCommand('server') + .description('create a P2P shell server') + .option('-f ', 'filename of the server seed key') + .option('--firewall ', 'list of allowed public keys') + .option('--disable-firewall', 'allow anyone to connect', false) + .option('--protocol ', 'list of allowed protocols') + .option('--tunnel-host ', 'restrict tunneling to a limited set of hosts') + .option('--tunnel-port ', 'restrict tunneling to a limited set of ports') + .option('--bootstrap ', 'custom dht nodes') + .action(require('../lib/bin/server.js')) diff --git a/constants.js b/constants.js deleted file mode 100644 index a9100e7..0000000 --- a/constants.js +++ /dev/null @@ -1,6 +0,0 @@ -const os = require('os') -const path = require('path') - -module.exports = { - SHELLDIR: path.join(os.homedir(), '.hypershell') -} diff --git a/index.js b/index.js new file mode 100644 index 0000000..95f5d34 --- /dev/null +++ b/index.js @@ -0,0 +1,208 @@ +const DHT = require('hyperdht') +const Protomux = require('protomux') +const HypercoreId = require('hypercore-id-encoding') +const crypto = require('hypercore-crypto') +const { ShellServer, ShellClient } = require('./lib/protocols/shell.js') + +module.exports = class Hypershell { + constructor (opts = {}) { + this.dht = opts.dht || new DHT({ bootstrap: opts.bootstrap }) + + this._autoDestroy = !opts.dht + } + + createServer (opts = {}) { + return new Server(this.dht, { + ...opts, + onsocket: function (socket) { + const mux = Protomux.from(socket) + + mux.pair({ protocol: 'hypershell' }, function () { + ShellServer.attach(mux) + }) + } + }) + } + + login (publicKey, opts = {}) { + const client = new Client(this.dht, publicKey, opts) + + return new ShellClient(client.socket, { + rawArgs: opts.rawArgs, + stdin: opts.stdin, + stdout: opts.stdout + }) + } + + async destroy () { + if (this._autoDestroy) { + await this.dht.destroy() + } + } +} + +class Server { + constructor (dht, opts = {}) { + this.dht = dht + this.keyPair = opts.keyPair || crypto.keyPair(opts.seed) + this.firewall = opts.firewall || opts.firewall === null ? opts.firewall : [] + this.verbose = !!opts.verbose + + this._server = this.dht.createServer({ + firewall: this._onFirewall.bind(this) + }) + + this._server.on('connection', this._onConnection.bind(this)) + + this._connections = new Set() + this._onsocket = opts.onsocket || null + } + + async listen (keyPair) { + await this._server.listen(keyPair || this.keyPair) + } + + async close () { + await this._server.close() // TODO: Force option? + + await closeConnections(this._connections, true) + } + + get publicKey () { + return this.keyPair.publicKey + } + + _onConnection (socket) { + this._connections.add(socket) + + socket.setKeepAlive(5000) + + socket.on('end', function () { + socket.end() + }) + + socket.on('error', function (err) { + if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') { + return + } + + // TODO + console.error(err.code, err) + }) + + socket.on('close', () => { + this._connections.delete(socket) + + if (this.verbose) { + console.log('Connection closed', HypercoreId.encode(socket.remotePublicKey)) + } + }) + + if (this.verbose) { + console.log('Connection opened', HypercoreId.encode(socket.remotePublicKey)) + } + + if (this._onsocket) { + this._onsocket(socket) + } + } + + _onFirewall (remotePublicKey, remoteHandshakePayload) { + if (this.firewall === null) { + return false + } + + for (const publicKey of this.firewall) { + if (remotePublicKey.equals(publicKey)) { + return false + } + } + + if (this.verbose) { + console.log('Firewall denied', HypercoreId.encode(remotePublicKey)) + } + + return true + } +} + +class Client { + constructor (dht, publicKey, opts = {}) { + this.dht = dht + this.keyPair = opts.keyPair || crypto.keyPair(opts.seed) + this.verbose = !!opts.verbose + this.inherit = !!opts.inherit + + this.socket = this.dht.connect(publicKey, { + keyPair: this.keyPair, + reusableSocket: opts.reusableSocket + }) + + this._onerror = opts.onerror || null + + this._open() + } + + _open () { + this.socket.setKeepAlive(5000) + + this.socket.on('error', this._onSocketError.bind(this)) + this.socket.on('end', this._onSocketEnd.bind(this)) + this.socket.on('close', this._onSocketClose.bind(this)) + } + + _onSocketEnd () { + this.socket.end() + } + + _onSocketError (err) { + if (this._onerror) { + this._onerror(err) + } + + if (this.inherit) { + process.exitCode = 1 + } + + if (!this.verbose) { + return + } + + if (err.code === 'ECONNRESET') console.error('Connection closed.') + else if (err.code === 'ETIMEDOUT') console.error('Connection timed out.') + else if (err.code === 'PEER_NOT_FOUND') console.error(err.message) + else if (err.code === 'PEER_CONNECTION_FAILED') console.error(err.message, '(probably firewalled)') + else console.error(err) + } + + _onSocketClose () { + // TODO: Improve by removing listeners etc + // this.close().catch(safetyCatch) + } +} + +function closeConnections (sockets, force) { + return new Promise(resolve => { + if (sockets.size === 0) { + resolve() + return + } + + let waiting = 0 + + for (const socket of sockets) { + waiting++ + + socket.on('close', onclose) + + if (force) socket.destroy() + else socket.end() + } + + function onclose () { + if (--waiting === 0) { + resolve() + } + } + }) +} diff --git a/lib/bin/keygen.js b/lib/bin/keygen.js new file mode 100644 index 0000000..4aad0c8 --- /dev/null +++ b/lib/bin/keygen.js @@ -0,0 +1,42 @@ +const fs = require('fs') +const path = require('path') +const crypto = require('hypercore-crypto') +const HypercoreId = require('hypercore-id-encoding') +const constants = require('../constants.js') +const question = require('../question.js') + +module.exports = async function keygen (opts = {}) { + console.log(opts) + let { + f: filename = path.join(constants.dir, 'id'), + comment = opts.comment ? (' # ' + opts.comment) : '' + } = opts + + console.log('Generating key.', { filename }) + + if (!opts.f) { + const answer = await question('Enter file in which to save the key (' + filename + '): ') + + if (answer) { + filename = answer + } + } + + filename = path.resolve(filename) + + if (fs.existsSync(filename)) { + throw new Error('File already exists:' + filename) + } + + const seed = crypto.randomBytes(32) + const keyPair = crypto.keyPair(seed) + + await fs.promises.mkdir(path.dirname(filename), { recursive: true }) + await fs.promises.writeFile(filename, HypercoreId.encode(seed) + comment + '\n', { flag: 'wx', mode: '600' }) + + console.log('Your key has been saved in', filename) + console.log('The public key is:') + console.log(HypercoreId.encode(keyPair.publicKey)) + + return keyPair +} diff --git a/lib/bin/login.js b/lib/bin/login.js new file mode 100644 index 0000000..8700f6f --- /dev/null +++ b/lib/bin/login.js @@ -0,0 +1,35 @@ +const fs = require('fs') +const keygen = require('./keygen.js') +const Hypershell = require('../../index.js') +const getKnownPeer = require('../get-known-peer.js') +const fileToKeyPair = require('../file-to-keypair.js') + +module.exports = async function login (serverPublicKey, opts = {}) { + console.log('login', opts) + + if (!fs.existsSync(opts.f)) { + await keygen({ filename: opts.f }) + } + + const target = await getKnownPeer(serverPublicKey, { verbose: true }) + + const hs = new Hypershell({ + bootstrap: opts.bootstrap + }) + + const shell = hs.login(target, { + keyPair: await fileToKeyPair(opts.f), + rawArgs: this.rawArgs, + stdin: process.stdin, + stdout: process.stdout, + verbose: true, + inherit: true + /* onerror: function (err) { + process.exitCode = 1 + } */ + }) + + await shell.channel.fullyClosed() + + await hs.destroy() +} diff --git a/lib/bin/server.js b/lib/bin/server.js new file mode 100644 index 0000000..e8f9ddf --- /dev/null +++ b/lib/bin/server.js @@ -0,0 +1,103 @@ +const fs = require('fs') +const path = require('path') +const goodbye = require('graceful-goodbye') +const readFile = require('read-file-live') +const HypercoreId = require('hypercore-id-encoding') +const configs = require('tiny-configs') +const constants = require('../constants.js') +const keygen = require('./keygen.js') +const Hypershell = require('../../index.js') +const fileToKeyPair = require('../file-to-keypair.js') + +const PROTOCOLS = ['shell', 'upload', 'download', 'tunnel'] + +module.exports = async function server (opts = {}) { + console.log('server', opts) + + const keyFilename = path.resolve(opts.f || path.join(constants.dir, 'id')) + const firewallFilename = path.resolve(opts.firewall || path.join(constants.dir, 'authorized_peers')) + const firewallEnabled = !opts.disableFirewall + const protocols = opts.protocol || PROTOCOLS + + if (!fs.existsSync(keyFilename)) { + await keygen({ filename: keyFilename }) + } + + if (firewallEnabled && !fs.existsSync(firewallFilename)) { + console.log('Notice: creating default firewall', firewallFilename) + + await fs.promises.mkdir(path.dirname(firewallFilename), { recursive: true }) + await fs.promises.writeFile(firewallFilename, '# \n', { flag: 'wx' }) + } + + const hs = new Hypershell({ + bootstrap: opts.bootstrap + }) + + const server = hs.createServer({ + keyPair: await fileToKeyPair(keyFilename), + verbose: true + }) + + let unregisterFirewall = null + + if (opts.disableFirewall) { + server.firewall = null + } else { + unregisterFirewall = await handleFirewall(firewallFilename, function (keys) { + server.firewall = keys + }) + } + + await server.listen() + + if (protocols === PROTOCOLS) { + console.log('To connect to this shell, on another computer run:') + console.log('hypershell ' + HypercoreId.encode(server.publicKey)) + } else { + console.log('Running server with restricted protocols') + console.log('Server key: ' + HypercoreId.encode(server.publicKey)) + } + console.log() + + const unregister = goodbye(close) + + return async function cleanup () { + unregister() + + await close() + } + + async function close () { + await server.close() + await hs.destroy() + + if (unregisterFirewall) { + unregisterFirewall() + } + } +} + +async function handleFirewall (filename, onchange) { + let list = null + + try { + list = await fs.promises.readFile(filename, 'utf8') + } catch (err) { + if (err.code !== 'ENOENT') { + throw err + } + } + + onchange(read(list)) + + return readFile(filename, buf => { + onchange(read(buf)) + }) + + function read (list) { + const parsed = configs.parse(list) + + return parsed.map(v => HypercoreId.decode(v)) + } +} diff --git a/lib/client-socket.js b/lib/client-socket.js deleted file mode 100644 index fce7fbd..0000000 --- a/lib/client-socket.js +++ /dev/null @@ -1,67 +0,0 @@ -const fs = require('fs') -const DHT = require('hyperdht') -const goodbye = require('graceful-goodbye') -const HypercoreId = require('hypercore-id-encoding') -const getKnownPeer = require('./get-known-peer.js') - -module.exports = { - ClientSocket, - waitForSocketTermination -} - -function ClientSocket ({ keyfile, serverPublicKey, reusableSocket = false, testnet = false }) { - serverPublicKey = getKnownPeer(serverPublicKey) - - const seed = HypercoreId.decode(fs.readFileSync(keyfile, 'utf8').trim()) - const keyPair = DHT.keyPair(seed) - - const node = new DHT({ bootstrap: testnet ? [{ host: '127.0.0.1', port: 40838 }] : undefined }) - const unregisterNode = goodbye(() => node.destroy(), 2) - - const socket = node.connect(serverPublicKey, { keyPair, reusableSocket }) - const unregisterSocket = goodbye(() => { - socket.end() - return waitForSocketTermination(socket) - }, 1) - - socket.on('error', function (error) { - if (error.code === 'ECONNRESET') console.error('Connection closed.') - else if (error.code === 'ETIMEDOUT') console.error('Connection timed out.') - else if (error.code === 'PEER_NOT_FOUND') console.error(error.message) - else if (error.code === 'PEER_CONNECTION_FAILED') console.error(error.message, '(probably firewalled)') - else console.error(error) - - process.exitCode = 1 - }) - - socket.on('end', () => socket.end()) - socket.once('close', () => node.destroy()) - socket.once('close', () => unregisterNode()) - socket.once('close', () => unregisterSocket()) - - socket.setKeepAlive(5000) - - return { node, socket } -} - -function waitForSocketTermination (socket) { - return new Promise((resolve) => { - const isClosed = socket.rawStream._closed - const isReadableEnded = socket.rawStream._readableState.ended - const isWritableEnded = socket.rawStream._writableState.ended - - if (isClosed || (isReadableEnded && isWritableEnded)) { - resolve() - return - } - - socket.on('end', onterm) - socket.on('close', onterm) - - function onterm () { - socket.removeListener('end', onterm) - socket.removeListener('close', onterm) - resolve() - } - }) -} diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..9ba275c --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,4 @@ +const os = require('os') +const path = require('path') + +exports.dir = path.join(os.homedir(), '.hypershell') diff --git a/lib/download.js b/lib/download.js deleted file mode 100644 index 85befd5..0000000 --- a/lib/download.js +++ /dev/null @@ -1,141 +0,0 @@ -const fs = require('fs') -const path = require('path') -const c = require('compact-encoding') -const m = require('../messages.js') -const tar = require('tar-fs') -const os = require('os') - -const EMPTY = Buffer.alloc(0) - -class DownloadServer { - constructor ({ mux }) { - this.channel = mux.createChannel({ - protocol: 'hypershell-download', - id: null, - handshake: m.handshakeDownload, - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this), - messages: [ - { encoding: m.downloadHeader }, // header - { encoding: m.error }, // errors - { encoding: c.raw } // data - ] - }) - - this.pack = null - } - - open () { - this.channel.open({}) - } - - onopen (handshake) { - const { source } = handshake - - try { - const st = fs.lstatSync(source) - this.channel.messages[0].send({ isDirectory: st.isDirectory() }) - } catch (error) { - this.channel.messages[1].send(error) - this.channel.close() - return - } - - this.pack = tar.pack(source) - - this.pack.once('error', (error) => { - this.channel.messages[1].send(error) - this.channel.close() - }) - - this.pack.on('data', (chunk) => this.channel.messages[2].send(chunk)) - this.pack.once('end', () => this.channel.messages[2].send(EMPTY)) - } - - onclose () { - if (this.pack) this.pack.destroy() - } -} - -class DownloadClient { - constructor ({ sourcePath, targetPath }, { socket, mux }) { - this.sourcePath = sourcePath - this.targetPath = path.resolve(resolveHomedir(targetPath)) - - this.socket = socket - - this.channel = mux.createChannel({ - protocol: 'hypershell-download', - id: null, - handshake: m.handshakeDownload, - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this), - messages: [ - { encoding: m.downloadHeader, onmessage: this.ondownloadheader.bind(this) }, // header - { encoding: m.error, onmessage: this.ondownloaderror.bind(this) }, // errors - { encoding: c.raw, onmessage: this.ondownload.bind(this) } // data - ] - }) - - this.extract = null - } - - open () { - this.channel.open({ source: this.sourcePath }) - } - - onopen () {} - - onclose () { - this.socket.end() - - if (this.extract) this.extract.destroy() - } - - ondownloadheader (data, c) { - const { isDirectory } = data - - const dir = isDirectory ? this.targetPath : path.dirname(this.targetPath) - const opts = { - readable: true, - writable: true, - map: (header) => { - if (!isDirectory) header.name = path.basename(this.targetPath) - return header - } - } - - this.extract = tar.extract(dir, opts) - - this.extract.once('error', function (error) { - console.error(error) - c.close() - }) - - this.extract.once('finish', function () { - c.close() - }) - } - - ondownloaderror (data, c) { - if (data.code === 'ENOENT') console.error('hypershell-server:', data.path + ': No such file or directory') - else console.error(data.message) - - c.close() - } - - ondownload (data, c) { - if (data.length) this.extract.write(data) - else this.extract.end() - } -} - -module.exports = { DownloadServer, DownloadClient } - -// Based on expand-home-dir -function resolveHomedir (str) { - if (!str) return str - if (str === '~') return os.homedir() - if (str.slice(0, 2) !== '~/') return str - return path.join(os.homedir(), str.slice(2)) -} diff --git a/lib/file-to-keypair.js b/lib/file-to-keypair.js new file mode 100644 index 0000000..e3d2c02 --- /dev/null +++ b/lib/file-to-keypair.js @@ -0,0 +1,11 @@ +const fs = require('fs') +const crypto = require('hypercore-crypto') +const HypercoreId = require('hypercore-id-encoding') + +module.exports = async function fileToKeyPair (filename) { + const key = await fs.promises.readFile(filename, 'utf8') + const seed = HypercoreId.decode(key.trim()) + const keyPair = crypto.keyPair(seed) + + return keyPair +} diff --git a/lib/get-known-peer.js b/lib/get-known-peer.js index 7db1829..43cd43e 100644 --- a/lib/get-known-peer.js +++ b/lib/get-known-peer.js @@ -2,10 +2,12 @@ const fs = require('fs') const path = require('path') const configs = require('tiny-configs') const HypercoreId = require('hypercore-id-encoding') -const { SHELLDIR } = require('../constants.js') +const constants = require('./constants.js') -module.exports = function getKnownPeer (host) { - for (const peer of readKnownPeers()) { +module.exports = async function getKnownPeer (host, opts) { + const peers = await readKnownPeers(opts) + + for (const peer of peers) { if (peer.name === host) { host = peer.publicKey break @@ -15,21 +17,28 @@ module.exports = function getKnownPeer (host) { return HypercoreId.decode(host) } -function readKnownPeers () { - const filename = path.join(SHELLDIR, 'known_peers') +async function readKnownPeers (opts) { + const filename = path.join(constants.dir, 'known_peers') if (!fs.existsSync(filename)) { - // console.log('Notice: creating default known peers', filename) - fs.mkdirSync(path.dirname(filename), { recursive: true }) - fs.writeFileSync(filename, '# \n', { flag: 'wx' }) + if (opts && opts.verbose) { + console.log('Notice: creating default known peers', filename) + } + + await fs.promises.mkdir(path.dirname(filename), { recursive: true }) + await fs.promises.writeFile(filename, '# \n', { flag: 'wx' }) } try { - const file = fs.readFileSync(filename, 'utf8') + const file = await fs.promises.readFile(filename, 'utf8') + return configs.parse(file, { split: ' ', length: 2 }) .map(m => ({ name: m[0], publicKey: m[1] })) } catch (error) { - if (error.code === 'ENOENT') return [] + if (error.code === 'ENOENT') { + return [] + } + throw error } } diff --git a/lib/local-tunnel.js b/lib/local-tunnel.js deleted file mode 100644 index 8e40d4e..0000000 --- a/lib/local-tunnel.js +++ /dev/null @@ -1,250 +0,0 @@ -const net = require('net') -const c = require('compact-encoding') -const pump = require('pump') -const DHT = require('hyperdht') -const Protomux = require('protomux') -const SecretStream = require('@hyperswarm/secret-stream') -const ReadyResource = require('ready-resource') -const safetyCatch = require('safety-catch') - -class LocalTunnelServer { - constructor ({ node, socket, mux, options }) { - this.dht = node - this.socket = socket - - this.channel = mux.createChannel({ - protocol: 'hypershell-tunnel-local', - id: null, - handshake: c.json, - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this), - messages: [ - { encoding: c.json, onmessage: this.onstreamid.bind(this) } - ] - }) - - this.options = options - - this.streams = new Map() - this.config = null - } - - open () { - this.channel.open({}) - } - - onopen (handshake) { - const isHostAllowed = LocalTunnelServer.firewallHosts(this.options.tunnelHost, handshake.host) - const isPortAllowed = LocalTunnelServer.firewallPorts(this.options.tunnelPort, handshake.port) - - if (!isHostAllowed || !isPortAllowed) { - this.channel.close() - return - } - - this.config = handshake - } - - onclose () { - for (const [, stream] of this.streams) { - stream.destroy() - } - } - - onstreamid (data, c) { - const { clientId } = data - - const rawStream = this.dht.createRawStream() - this.streams.set(rawStream.id, rawStream) - rawStream.on('close', () => this.streams.delete(rawStream.id)) - rawStream.on('error', safetyCatch) - - c.messages[0].send({ clientId, serverId: rawStream.id }) - - DHT.connectRawStream(this.socket, rawStream, clientId) - const secretStream = new SecretStream(true, rawStream) - secretStream.on('error', safetyCatch) - - secretStream.setKeepAlive(5000) - - const remoteSocket = net.connect(this.config.port, this.config.host) - rawStream.userData = { remoteSocket, secretStream } - - pump(secretStream, remoteSocket, secretStream) - } - - static firewallHosts (hosts, target) { - if (!hosts) return true - - for (const host of hosts) { - // + support for CIDR ranges? - if (host === target) return true - } - - return false - } - - static firewallPorts (ports, target) { - if (!ports) return true - - for (const port of ports) { - const isRange = port.indexOf('-') > -1 - let list = null - - if (isRange) { - const [start, end] = port.split('-', 2).map(Number) - const length = end - start + 1 - - list = Array.from({ length }, (_, i) => start + i) - } else { - list = [Number(port)] - } - - for (const number of list) { - if (number === target) return true - } - } - - return false - } -} - -class LocalTunnelClient extends ReadyResource { - constructor (config, { node, keyPair, serverPublicKey }) { - super() - - this.dht = node - - this.keyPair = keyPair - this.serverPublicKey = serverPublicKey - - this.config = LocalTunnelClient.parse(config) // + defaults - - this.streams = new Map() - this.server = net.createServer(this.onconnection.bind(this)) // + option for udp - - this.ready().catch(safetyCatch) - } - - async _open () { - this.server.listen(this.config.local.port, this.config.local.host) - - await waitForServer(this.server) - } - - _close () { - this.server.close() - - if (this.mux) this.mux.destroy() - } - - _createMux () { - if (this.mux && !this.mux.stream.destroying) return - - // + reusableSocket for when having several -L tunnels? - const socket = this.dht.connect(this.serverPublicKey, { keyPair: this.keyPair }) - - socket.setKeepAlive(5000) - - this.mux = new Protomux(socket) - } - - _createChannel () { - if (this.mux.opened({ protocol: 'hypershell-tunnel-local', id: null })) return - - const channel = this.mux.createChannel({ - protocol: 'hypershell-tunnel-local', - id: null, - handshake: c.json, - messages: [ - { encoding: c.json, onmessage: this.onstreamid.bind(this) } - ], - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this) - }) - - if (channel === null) return - - this.channel = channel - this.channel.open(this.config.remote) - } - - onopen () { - // No-op - } - - onclose () { - for (const [, stream] of this.streams) { - stream.destroy() - } - } - - onconnection (localSocket) { - this._createMux() - this._createChannel() - - const rawStream = this.dht.createRawStream() - this.streams.set(rawStream.id, rawStream) - rawStream.on('close', () => this.streams.delete(rawStream.id)) - rawStream.on('error', safetyCatch) - - rawStream.userData = { localSocket } - rawStream.on('close', () => localSocket.destroy()) - localSocket.on('error', safetyCatch) - - this.channel.messages[0].send({ clientId: rawStream.id, serverId: 0 }) - } - - onstreamid (data, channel) { - const { clientId, serverId } = data - - const rawStream = this.streams.get(clientId) - if (!rawStream) throw new Error('Stream not found: ' + clientId) - const { localSocket } = rawStream.userData - - DHT.connectRawStream(this.mux.stream, rawStream, serverId) - const secretStream = new SecretStream(false, rawStream) - secretStream.on('error', safetyCatch) - - secretStream.setKeepAlive(5000) - - rawStream.userData.secretStream = secretStream - - pump(localSocket, secretStream, localSocket) - } - - static parse (config) { - const match = config.match(/(?:(.*):)?([\d]+):(?:(.*):)?([\d]+)/i) - - // + should return errors - if (!match[2]) errorAndExit('local port is required') - if (!match[3]) errorAndExit('remote host is required') - if (!match[4]) errorAndExit('remote port is required') - - const local = { host: match[1] || '0.0.0.0', port: Number(match[2]) } - const remote = { host: match[3], port: Number(match[4]) } - - return { local, remote } - } -} - -function waitForServer (server) { - return new Promise((resolve, reject) => { - server.on('listening', done) - server.on('error', done) - // if (server.listening) done() - - function done (error) { - server.removeListener('listening', done) - server.removeListener('error', done) - error ? reject(error) : resolve() - } - }) -} - -module.exports = { LocalTunnelServer, LocalTunnelClient } - -function errorAndExit (message) { - console.error('Error:', message) - process.exit(1) -} diff --git a/messages.js b/lib/protocols/messages.js similarity index 100% rename from messages.js rename to lib/protocols/messages.js diff --git a/lib/shell.js b/lib/protocols/shell.js similarity index 51% rename from lib/shell.js rename to lib/protocols/shell.js index 15aa9fb..9c1ba45 100644 --- a/lib/shell.js +++ b/lib/protocols/shell.js @@ -1,14 +1,17 @@ const os = require('os') +const Protomux = require('protomux') const c = require('compact-encoding') -const m = require('../messages.js') const PTY = require('tt-native') +const { PassThrough } = require('streamx') +const m = require('./messages.js') +const waitForSocket = require('../wait-for-socket.js') const isWin = os.platform() === 'win32' const shellFile = isWin ? 'powershell.exe' : (process.env.SHELL || 'bash') const EMPTY = Buffer.alloc(0) class ShellServer { - constructor ({ mux }) { + constructor (mux) { this.channel = mux.createChannel({ protocol: 'hypershell', id: null, @@ -24,13 +27,19 @@ class ShellServer { ] }) + if (!this.channel) { + throw new Error('Channel duplicated') + } + this.pty = null - } - open () { this.channel.open({}) } + static attach (mux) { + return new this(mux) + } + onopen (handshake) { try { this.pty = PTY.spawn(handshake.command || shellFile, handshake.args, { @@ -39,9 +48,9 @@ class ShellServer { width: handshake.width, height: handshake.height }) - } catch (error) { + } catch (err) { this.channel.messages[3].send(1) - this.channel.messages[2].send(Buffer.from(error.toString() + '\n')) + this.channel.messages[2].send(Buffer.from(err.toString() + '\n')) this.channel.close() return } @@ -70,14 +79,16 @@ class ShellServer { } class ShellClient { - constructor (rawArgs, { socket, mux }) { - const spawn = ShellClient.parseVariadic(rawArgs) + constructor (socket, opts = {}) { + const spawn = parseVariadic(opts.rawArgs || []) + this.command = spawn.shift() || '' this.args = spawn this.socket = socket + this.mux = Protomux.from(socket) - this.channel = mux.createChannel({ + this.channel = this.mux.createChannel({ protocol: 'hypershell', id: null, handshake: m.handshakeSpawn, @@ -91,75 +102,118 @@ class ShellClient { { encoding: m.resize } ] }) - } - open () { + if (!this.channel) { + throw new Error('Channel duplicated') + } + + this.stdin = opts.stdin || new PassThrough() + this.stdout = opts.stdout || new PassThrough() + this.exitCode = null + + this.onstdinBound = this.onstdin.bind(this) + this.onresizeBound = this.onresize.bind(this) + this.onsocketcloseBound = this.onsocketclose.bind(this) + this.channel.open({ command: this.command, args: this.args, - width: process.stdout.columns || 80, // cols/rows doesn't exists if spawned without a terminal - height: process.stdout.rows || 24 + width: this.stdout.columns || 80, // cols/rows doesn't exists if spawned without a terminal + height: this.stdout.rows || 24 }) - this.setup() + this._setup() + } + + async ready () { + await this.channel.fullyOpened() + } + + async close () { + this.socket.destroy() + + // TODO: Not needed anymore? + await waitForSocket(this.socket) + + await this.channel.fullyClosed() + } + + destroy () { + this.socket.destroy() } onopen () {} onclose () { - this.socket.end() + this.socket.destroy() } - setup () { - this.onstdin = this.onstdin.bind(this) - this.onresize = this.onresize.bind(this) - this.onsocketclose = this.onsocketclose.bind(this) + _setup () { + if (this.stdin.isTTY) { + this.stdin.setRawMode(true) + } - if (process.stdin.isTTY) process.stdin.setRawMode(true) - process.stdin.on('data', this.onstdin) // + stdin 'end' event? - process.stdout.on('resize', this.onresize) - this.socket.on('close', this.onsocketclose) + this.stdin.on('data', this.onstdinBound) + this.stdout.on('resize', this.onresizeBound) + this.socket.on('close', this.onsocketcloseBound) - process.stdin.resume() + this.stdin.resume() } onstdin (data) { + if (typeof data === 'string') { + data = Buffer.from(data) + } + this.channel.messages[0].send(data) } onstdout (data, c) { - process.stdout.write(data) + this.stdout.write(data) } onstderr (data, c) { - process.stderr.write(data) + this.stderr.write(data) } onexitcode (code, c) { - process.exitCode = code + this.exitCode = code + + // TODO + if (this.stdin === process.stdin) { + process.exitCode = code + } } onresize () { this.channel.messages[4].send({ - width: process.stdout.columns || 80, - height: process.stdout.rows || 24 + width: this.stdout.columns || 80, + height: this.stdout.rows || 24 }) } onsocketclose () { - if (process.stdin.isTTY) process.stdin.setRawMode(false) - process.stdin.removeListener('data', this.onstdin) - process.stdout.removeListener('resize', this.onresize) - this.socket.removeListener('close', this.onsocketclose) + if (this.stdin.isTTY) { + this.stdin.setRawMode(false) + } - process.stdin.pause() // + process.exit()? - } + this.stdin.removeListener('data', this.onstdinBound) + this.stdout.removeListener('resize', this.onresizeBound) + this.socket.removeListener('close', this.onsocketcloseBound) - static parseVariadic (rawArgs) { - const index = rawArgs.indexOf('--') - const variadic = index === -1 ? null : rawArgs.splice(index + 1) - return variadic || [] + this.stdin.pause() } } -module.exports = { ShellServer, ShellClient, shellFile } +module.exports = { + ShellServer, + ShellClient, + shellFile +} + +function parseVariadic (rawArgs) { + const index = rawArgs.indexOf('--') + const variadic = index === -1 ? null : rawArgs.splice(index + 1) + + return variadic || [] +} diff --git a/lib/question.js b/lib/question.js new file mode 100644 index 0000000..5a3d877 --- /dev/null +++ b/lib/question.js @@ -0,0 +1,16 @@ +const readline = require('readline') + +module.exports = function question (query) { + return new Promise(resolve => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }) + + rl.question(query, function (answer) { + rl.close() + + resolve(answer.trim()) + }) + }) +} diff --git a/lib/upload.js b/lib/upload.js deleted file mode 100644 index 43ff054..0000000 --- a/lib/upload.js +++ /dev/null @@ -1,131 +0,0 @@ -const fs = require('fs') -const path = require('path') -const c = require('compact-encoding') -const m = require('../messages.js') -const tar = require('tar-fs') -const os = require('os') - -const EMPTY = Buffer.alloc(0) - -class UploadServer { - constructor ({ mux }) { - this.channel = mux.createChannel({ - protocol: 'hypershell-upload', - id: null, - handshake: m.handshakeUpload, - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this), - messages: [ - null, // no header - { encoding: m.error }, // errors - { encoding: c.raw, onmessage: this.onupload.bind(this) } // data - ] - }) - } - - open () { - this.channel.open({}) - } - - onopen (handshake) { - const { target, isDirectory } = handshake - - const dir = isDirectory ? target : path.dirname(target) - this.extract = tar.extract(dir, { - readable: true, - writable: true, - map: (header) => { - if (!isDirectory) header.name = path.basename(target) - return header - } - }) - - this.extract.once('error', (error) => { - this.channel.messages[1].send(error) - this.channel.close() - }) - - this.extract.once('finish', () => this.channel.close()) - } - - onclose () { - if (this.extract) this.extract.destroy() - } - - onupload (data, c) { - if (data.length) this.extract.write(data) - else this.extract.end() - } -} - -class UploadClient { - constructor ({ sourcePath, targetPath }, { socket, mux }) { - this.sourcePath = path.resolve(resolveHomedir(sourcePath)) - this.targetPath = targetPath - - this.socket = socket - - this.channel = mux.createChannel({ - protocol: 'hypershell-upload', - id: null, - handshake: m.handshakeUpload, - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this), - messages: [ - null, // no header - { encoding: m.error, onmessage: this.onuploaderror.bind(this) }, // errors - { encoding: c.raw } // data - ] - }) - } - - open () { - try { - const st = fs.lstatSync(this.sourcePath) - - this.channel.open({ - target: this.targetPath, - isDirectory: st.isDirectory() - }) - } catch (error) { - if (error.code === 'ENOENT') console.error(this.sourcePath + ': No such file or directory') - else console.error(error.message) - - this.socket.destroy() - return - } - - this.pack = tar.pack(this.sourcePath) - - this.pack.once('error', (error) => { - console.error(error.message) - this.channel.close() - }) - - this.pack.on('data', (chunk) => this.channel.messages[2].send(chunk)) - this.pack.once('end', () => this.channel.messages[2].send(EMPTY)) - } - - onopen () {} - - onclose () { - this.socket.end() - - if (this.pack) this.pack.destroy() - } - - onuploaderror (data, c) { - console.error('hypershell-server:', data) - c.close() - } -} - -module.exports = { UploadServer, UploadClient } - -// Based on expand-home-dir -function resolveHomedir (str) { - if (!str) return str - if (str === '~') return os.homedir() - if (str.slice(0, 2) !== '~/') return str - return path.join(os.homedir(), str.slice(2)) -} diff --git a/package.json b/package.json index ac68e87..08511f0 100644 --- a/package.json +++ b/package.json @@ -4,44 +4,39 @@ "description": "Spawn shells anywhere. Fully peer-to-peer, authenticated, and end to end encrypted", "main": "index.js", "bin": { - "hypershell-server": "./bin/server.js", - "hypershell": "./bin/client.js", - "hypershell-copy": "./bin/copy.js", - "hypershell-keygen": "./bin/keygen.js" + "hypershell": "bin.js" }, "scripts": { - "test": "standard && brittle test/*.js", - "lint": "standard" + "test": "standard && brittle test/*.js" }, "repository": { "type": "git", - "url": "https://github.com/holepunchto/hypershell.git" + "url": "git+https://github.com/holepunchto/hypershell.git" }, "author": "Lucas Barrena (@LuKks)", "license": "Apache-2.0", "bugs": { "url": "https://github.com/holepunchto/hypershell/issues" }, - "homepage": "https://github.com/holepunchto/hypershell", + "homepage": "https://github.com/holepunchto/hypershell#readme", + "devDependencies": { + "brittle": "^3.3.2", + "like-tmp": "^1.0.0", + "standard": "^17.1.0" + }, "dependencies": { - "@hyperswarm/secret-stream": "^6.1.0", - "commander": "^9.4.1", - "compact-encoding": "^2.11.0", - "graceful-goodbye": "^1.1.0", - "hypercore-id-encoding": "^1.2.0", - "hyperdht": "^6.6.0", - "keypear": "^1.1.1", - "protomux": "^3.4.0", - "pump": "^3.0.0", + "@hyperswarm/secret-stream": "^6.6.4", + "commander": "^12.1.0", + "compact-encoding": "^2.15.0", + "graceful-goodbye": "^1.3.0", + "hypercore-crypto": "^3.4.2", + "hypercore-id-encoding": "^1.3.0", + "hyperdht": "^6.19.0", + "protomux": "^3.10.0", + "protomux-rpc": "^1.6.0", + "pump": "^3.0.2", "read-file-live": "^1.0.1", - "ready-resource": "^1.0.0", - "safety-catch": "^1.0.2", - "tar-fs": "^2.1.1", "tiny-configs": "^1.1.0", - "tt-native": "^1.0.2" - }, - "devDependencies": { - "brittle": "^3.1.0", - "standard": "^17.0.0" + "tt-native": "^1.1.1" } } diff --git a/test/basic.js b/test/basic.js deleted file mode 100644 index 930fa9b..0000000 --- a/test/basic.js +++ /dev/null @@ -1,132 +0,0 @@ -const test = require('brittle') -const path = require('path') -const fs = require('fs') -const { create, spawnKeygen, spawnServer, spawnClient, spawnCopy, waitForProcess, waitForServerReady } = require('./helpers/index.js') -const { shellFile } = require('../lib/shell.js') - -test('keygen', async function (t) { - t.plan(6) - - const { root } = await create(t) - const keyfile = path.join(root, 'peer-random') - t.absent(fs.existsSync(keyfile)) - - const keygen = spawnKeygen(t, { keyfile }) - keygen.on('close', (code) => t.pass('keygen closed: ' + code)) - - keygen.stdout.on('data', (data) => { - if (data.indexOf('Generating key') > -1) t.pass('generating key') - if (data.indexOf('Your key has been saved') > -1) t.pass('key saved') - }) - - keygen.on('close', () => { - t.ok(fs.existsSync(keyfile)) - if (process.platform === 'win32') { - t.pass('No user-specific file permissions on windows') - } else { - const mode = fs.statSync(keyfile).mode.toString(8) - const permissions = mode.slice(-3) - t.is(permissions, '600') - } - }) - - await waitForProcess(keygen) -}) - -test('shell', async function (t) { - t.plan(4) - - const { clientkey, serverkey, firewall, serverKeyPair } = await create(t) - - const server = spawnServer(t, { serverkey, firewall }) - server.once('close', (code) => t.pass('server closed: ' + code)) - - server.stdout.on('data', (data) => { - if (data.startsWith('Firewall allowed:')) { - t.pass('Server firewall allowed') - } - }) - - await waitForServerReady(server) - - const client = spawnClient(t, serverKeyPair.publicKey.toString('hex'), { clientkey }) - client.on('close', (code) => t.pass('client closed: ' + code)) - - client.stdout.on('data', (data) => { - if (data.indexOf('The number is: 1234') > -1) { - t.pass('client stdout match') - - client.kill() - client.once('close', () => server.kill()) - } - }) - - await waitForProcess(client) - - if (shellFile.indexOf('powershell.exe') > -1) { - client.stdin.write(Buffer.from(' $env:HYPERSHELL_TEST_ENV="1234"\r\n echo "The number is: $env:HYPERSHELL_TEST_ENV"\r\n')) - } else { - client.stdin.write(Buffer.from(' HYPERSHELL_TEST_ENV="1234"\n echo "The number is: $HYPERSHELL_TEST_ENV"\n')) - } - // client.stdin.end() -}) - -test('copy - upload (absolute path)', async function (t) { - t.plan(5) - - const { root, clientkey, serverkey, firewall, serverKeyPair } = await create(t) - - const src = path.join(root, 'file-original.txt') - const dst = path.join(root, 'file-backup.txt') - - fs.writeFileSync(src, 'hello', { flag: 'wx' }) - t.absent(fs.existsSync(dst)) - - const server = spawnServer(t, { serverkey, firewall }) - server.once('close', (code) => t.pass('server closed: ' + code)) - await waitForServerReady(server) - - const pk = serverKeyPair.publicKey.toString('hex') - - const upload = spawnCopy(t, src, pk + ':' + dst, { clientkey }) - upload.on('close', (code) => t.pass('upload closed: ' + code)) - - upload.on('close', () => { - t.ok(fs.existsSync(dst)) - t.alike(fs.readFileSync(dst), Buffer.from('hello')) - - server.kill() - }) - - await waitForProcess(upload) -}) - -test('copy - download (absolute path)', async function (t) { - t.plan(5) - - const { root, clientkey, serverkey, firewall, serverKeyPair } = await create(t) - - const src = path.join(root, 'file-original.txt') - const dst = path.join(root, 'file-backup.txt') - - fs.writeFileSync(src, 'hello', { flag: 'wx' }) - t.absent(fs.existsSync(dst)) - - const server = spawnServer(t, { serverkey, firewall }) - server.once('close', (code) => t.pass('server closed: ' + code)) - await waitForServerReady(server) - - const pk = serverKeyPair.publicKey.toString('hex') - - const download = spawnCopy(t, pk + ':' + src, dst, { clientkey }) - download.on('close', (code) => t.pass('download closed: ' + code)) - - download.on('close', () => { - t.ok(fs.existsSync(dst)) - t.alike(fs.readFileSync(dst), Buffer.from('hello')) - - server.kill() - }) - - await waitForProcess(download) -}) diff --git a/test/bin.js b/test/bin.js new file mode 100644 index 0000000..165b6fe --- /dev/null +++ b/test/bin.js @@ -0,0 +1,61 @@ +const fs = require('fs') +const path = require('path') +const test = require('brittle') +const tmp = require('like-tmp') +const HypercoreId = require('hypercore-id-encoding') +const createTestnet = require('hyperdht/testnet') +const fileToKeyPair = require('../lib/file-to-keypair.js') +const { hypershell, hypershellSpawn, closeProcess, waitForLog } = require('./helpers/index.js') + +test('basic', async function (t) { + t.plan(1) + + const t2 = t.test('shell') + + t2.plan(1) + + const dir = await tmp(t) + const testnet = await createTestnet(3, t.teardown) + const bootstrap = testnet.bootstrap[0].host + ':' + testnet.bootstrap[0].port + + hypershell('keygen', ['-f', path.join(dir, 'server-key')]) + hypershell('keygen', ['-f', path.join(dir, 'client-key')]) + + const serverKeyPair = await fileToKeyPair(path.join(dir, 'server-key')) + const clientKeyPair = await fileToKeyPair(path.join(dir, 'client-key')) + + const server = await hypershellSpawn(t, 'server', [ + '--bootstrap', bootstrap, + '-f', path.join(dir, 'server-key'), + '--firewall', path.join(dir, 'firewall') + ]) + + await waitForLog(server, 'To connect to this shell') + + await fs.promises.writeFile(path.join(dir, 'firewall'), HypercoreId.encode(clientKeyPair.publicKey) + '\n') + + const shell = await hypershellSpawn(t, 'login', [ + HypercoreId.encode(serverKeyPair.publicKey), + '--bootstrap', bootstrap, + '-f', path.join(dir, 'client-key') + ]) + + let out = '' + + shell.stdout.on('data', function onstdout (data) { + out += data.toString() + + if (out.includes('Hello World!')) { + shell.stdout.removeListener('data', onstdout) + + t2.pass() + } + }) + + shell.stdin.write('echo "Hello World!"\n') + + await t2 + + await closeProcess(shell) + await closeProcess(server) +}) diff --git a/test/helpers/index.js b/test/helpers/index.js index 2a061f1..85eb104 100644 --- a/test/helpers/index.js +++ b/test/helpers/index.js @@ -1,141 +1,58 @@ const path = require('path') -const fs = require('fs') -const fsp = require('fs/promises') -const os = require('os') -const { spawn } = require('child_process') -// const { spawnSync } = require('child_process') -const createTestnet = require('hyperdht/testnet') -const DHT = require('hyperdht') -const Keychain = require('keypear') -const HypercoreId = require('hypercore-id-encoding') - -const BIN_KEYGEN = path.join(__dirname, '..', '..', 'bin/keygen.js') -const BIN_SERVER = path.join(__dirname, '..', '..', 'bin/server.js') -const BIN_CLIENT = path.join(__dirname, '..', '..', 'bin/client.js') -const BIN_COPY = path.join(__dirname, '..', '..', 'bin/copy.js') +const cp = require('child_process') -module.exports = { - BIN_KEYGEN, - BIN_SERVER, - BIN_CLIENT, - BIN_COPY, - create, - spawnServer, - spawnClient, - spawnCopy, - spawnKeygen, - keygen, - addAuthorizedPeer, - sleep, - waitForProcess, - waitForServerReady -} - -function createTmpDir (t) { - const tmpdir = path.join(os.tmpdir(), 'hypershell-test-') - const dir = fs.mkdtempSync(tmpdir) - t.teardown(() => fsp.rm(dir, { recursive: true })) - return dir -} - -async function create (t) { - const root = createTmpDir(t) - const clientkey = path.join(root, 'peer-client') - const serverkey = path.join(root, 'peer-server') - const firewall = path.join(root, 'authorized_peers') +const BIN_HYPERSHELL = path.join(__dirname, '..', '..', 'bin.js') - const clientKeyPair = keygen(clientkey) - const serverKeyPair = keygen(serverkey) - addAuthorizedPeer(firewall, clientkey) - - const swarm = await useTestnet(t) - - return { root, clientkey, serverkey, firewall, swarm, clientKeyPair, serverKeyPair } +module.exports = { + hypershell, + hypershellSpawn, + closeProcess, + waitForLog } -// + should require to pass the args array, and just automatically append --testnet - -function spawnKeygen (t, { keyfile }) { - const sp = spawn(process.execPath, [BIN_KEYGEN, '-f', keyfile], { timeout: 15000 }) - t.teardown(() => sp.kill()) +function hypershell (subcommand, args) { + args.unshift(subcommand) + args.unshift(BIN_HYPERSHELL) - sp.stdout.setEncoding('utf8') - sp.stderr.setEncoding('utf8') - - sp.on('error', (error) => t.fail('keygen error: ' + error.message)) - sp.stderr.on('data', (data) => t.fail('keygen stderr: ' + data)) - - return sp + return cp.execFileSync(process.execPath, args.filter(v => v), { encoding: 'utf8' }) } -function spawnServer (t, { serverkey, firewall }) { - const sp = spawn(process.execPath, [BIN_SERVER, '-f', serverkey, '--firewall', firewall, '--testnet'], { timeout: 15000 }) - t.teardown(() => sp.kill()) +async function hypershellSpawn (t, subcommand, args, opts = {}) { + args.unshift(subcommand) + args.unshift(BIN_HYPERSHELL) - sp.stdout.setEncoding('utf8') - sp.stderr.setEncoding('utf8') + const sp = cp.spawn(process.execPath, args, { timeout: 15000 }) - sp.on('error', (error) => t.fail('server error: ' + error.message)) - sp.stderr.on('data', (data) => t.fail('server stderr: ' + data)) + t.teardown(async () => { + if (sp.killed) return - return sp -} + sp.kill() -function spawnClient (t, serverPublicKey, { clientkey }) { - const sp = spawn(process.execPath, [BIN_CLIENT, serverPublicKey, '-f', clientkey, '--testnet'], { timeout: 15000 }) - t.teardown(() => sp.kill()) + await new Promise(resolve => sp.once('close', resolve)) + }) sp.stdout.setEncoding('utf8') sp.stderr.setEncoding('utf8') - sp.on('error', (error) => t.fail('client error: ' + error.message)) - sp.stderr.on('data', (data) => t.fail('client stderr: ' + data)) - - return sp -} - -function spawnCopy (t, source, target, { clientkey }) { - const sp = spawn(process.execPath, [BIN_COPY, source, target, '-f', clientkey, '--testnet'], { timeout: 15000 }) - t.teardown(() => sp.kill()) + sp.on('error', (error) => t.fail('spawn error: ' + error.message)) + sp.stderr.on('data', (data) => t.fail('spawn stderr: ' + data.toString())) - sp.stdout.setEncoding('utf8') - sp.stderr.setEncoding('utf8') + if (opts.verbose) { + sp.stdout.on('data', console.log) + sp.stderr.on('data', console.log) + } - sp.on('error', (error) => t.fail('copy error: ' + error.message)) - sp.stderr.on('data', (data) => t.fail('copy stderr: ' + data)) + await waitForProcess(sp) return sp } -function keygen (keyfile) { - const seed = Keychain.seed() - fs.mkdirSync(path.dirname(keyfile), { recursive: true }) - fs.writeFileSync(keyfile, HypercoreId.encode(seed) + '\n', { flag: 'wx' }) - return DHT.keyPair(seed) -} - -function addAuthorizedPeer (firewall, keyfile) { - const seed = HypercoreId.decode(fs.readFileSync(keyfile, 'utf8').trim()) - const keyPair = DHT.keyPair(seed) - if (!fs.existsSync(firewall)) fs.writeFileSync(firewall, '# \n', { flag: 'wx' }) - fs.appendFileSync(firewall, HypercoreId.encode(keyPair.publicKey) + '\n') -} +async function closeProcess (sp) { + if (sp.killed) return -async function useTestnet (t) { - const swarm = await createTestnet(3, { host: '127.0.0.1', port: 40838 }) - t.teardown(() => swarm.destroy()) + sp.kill() - const bootstrap = swarm.nodes[0].address() - if (bootstrap.port !== 40838) { - await swarm.destroy() - throw new Error('Swarm failed to be created on specific port') - } - - return swarm -} - -function sleep (ms) { - return new Promise(resolve => setTimeout(resolve, ms)) + await new Promise(resolve => sp.once('close', resolve)) } function waitForProcess (child) { @@ -151,10 +68,8 @@ function waitForProcess (child) { }) } -function waitForServerReady (child) { +function waitForLog (child, message) { return new Promise((resolve, reject) => { - let step = 0 - child.stdout.on('data', ondata) child.stderr.on('data', onstderror) child.on('error', onerror) @@ -166,13 +81,7 @@ function waitForServerReady (child) { } function ondata (data) { - const match1 = data.indexOf('To connect to this shell,') > -1 - if (match1) step++ - - const match2 = data.indexOf('hypershell ') > -1 - if (match2) step++ - - if (step === 2) { + if (data.includes(message)) { cleanup() resolve() } diff --git a/test/lib.js b/test/lib.js new file mode 100644 index 0000000..e9109b0 --- /dev/null +++ b/test/lib.js @@ -0,0 +1,75 @@ +const test = require('brittle') +const crypto = require('hypercore-crypto') +const { getStreamError } = require('streamx') +const createTestnet = require('hyperdht/testnet') +const Hypershell = require('../index.js') + +test('basic shell', async function (t) { + t.plan(2) + + const t2 = t.test('shell') + + t2.plan(1) + + const { bootstrap } = await createTestnet(3, t.teardown) + const hs = new Hypershell({ bootstrap }) + + const clientKeyPair = crypto.keyPair() + + const server = hs.createServer({ firewall: [clientKeyPair.publicKey] }) + await server.listen() + + const shell = hs.login(server.publicKey, { keyPair: clientKeyPair }) + + let out = '' + + shell.stdout.on('data', function onstdout (data) { + out += data.toString() + + if (out.includes('Hello World!')) { + shell.stdout.removeListener('data', onstdout) + + t2.pass() + } + }) + + shell.stdin.write('echo "Hello World!"\n') + + await t2 + + await shell.close() + await server.close() + await hs.destroy() + + t.absent(getStreamError(shell.socket)) +}) + +test('server is closed first', async function (t) { + const hs = await createHypershell(t) + const server = await createServer(hs) + const shell = hs.login(server.publicKey) + + await shell.ready() + + await server.close() + await shell.channel.fullyClosed() + await hs.destroy() + + const err = getStreamError(shell.socket) + + t.is(err.code, 'ECONNRESET') +}) + +async function createHypershell (t) { + const { bootstrap } = await createTestnet(3, t.teardown) + + return new Hypershell({ bootstrap }) +} + +async function createServer (hs, opts = {}) { + const server = hs.createServer({ firewall: null, ...opts }) + + await server.listen() + + return server +} From 380072bbb366e14d5ce538fae64c48d758d3a5ff Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Mon, 14 Oct 2024 06:14:28 -0300 Subject: [PATCH 02/21] Add copy --- bin.js | 1 + bin/copy.js | 72 +---------- bin/login.js | 4 +- index.js | 36 ++++++ lib/bin/copy.js | 58 +++++++++ lib/bin/keygen.js | 1 - lib/bin/login.js | 16 +-- lib/bin/server.js | 2 - lib/protocols/copy.js | 181 +++++++++++++++++++++++++++ lib/protocols/messages.js | 38 +++--- lib/protocols/tunnel.js | 250 ++++++++++++++++++++++++++++++++++++++ package.json | 1 + test/lib.js | 56 +++++++++ 13 files changed, 614 insertions(+), 102 deletions(-) mode change 100755 => 100644 bin/copy.js create mode 100644 lib/bin/copy.js create mode 100644 lib/protocols/copy.js create mode 100644 lib/protocols/tunnel.js diff --git a/bin.js b/bin.js index 748ef00..c620258 100755 --- a/bin.js +++ b/bin.js @@ -10,6 +10,7 @@ const main = program .addCommand(require('./bin/keygen.js')) .addCommand(require('./bin/server.js')) .addCommand(require('./bin/login.js')) + .addCommand(require('./bin/copy.js')) main.parseAsync().catch(err => { safetyCatch(err) diff --git a/bin/copy.js b/bin/copy.js old mode 100755 new mode 100644 index eb291be..b83a419 --- a/bin/copy.js +++ b/bin/copy.js @@ -1,70 +1,8 @@ -#!/usr/bin/env node +const { createCommand } = require('commander') -const fs = require('fs') -const path = require('path') -const Protomux = require('protomux') -const { Command } = require('commander') -const { SHELLDIR } = require('../constants.js') -const { ClientSocket } = require('../lib/client-socket.js') -const { UploadClient } = require('../lib/upload.js') -const { DownloadClient } = require('../lib/download.js') -const keygen = require('./keygen.js') - -const publicKeyExpr = /^([a-fA-F0-9]{64}|[ybndrfg8ejkmcpqxot1uwisza345h769]{52}):/i - -const program = new Command() - -program - .description('Transfers files using a P2P shell server as transport.') +module.exports = createCommand('copy') + .description('transfer files using a P2P server') .argument('', 'Source') .argument('', 'Target') - .option('-f ', 'Filename of the client seed key.', path.join(SHELLDIR, 'peer')) - // .option('--key ', 'Inline key for the client.') - .option('--testnet', 'Use a local testnet.', false) - .action(cmd) - .parseAsync() - -async function cmd (sourcePath, targetPath, options = {}) { - const fileOperation = sourcePath[0] === '@' || publicKeyExpr.test(sourcePath) ? 'download' : 'upload' - let serverPublicKey = null - - const keyfile = path.resolve(options.f) - - if (!fs.existsSync(keyfile)) { - await keygen({ f: keyfile }) - } - - if (sourcePath[0] === '@' || publicKeyExpr.test(sourcePath)) { - [serverPublicKey, sourcePath] = parseRemotePath(sourcePath) - if (!serverPublicKey || !sourcePath) errorAndExit('Invalid source path.') - } else if (targetPath[0] === '@' || publicKeyExpr.test(targetPath)) { - [serverPublicKey, targetPath] = parseRemotePath(targetPath) - if (!serverPublicKey || !targetPath) errorAndExit('Invalid target path.') - } else { - errorAndExit('Invalid source or target path.') - } - - const { node, socket } = ClientSocket({ keyfile, serverPublicKey, testnet: options.testnet }) - const mux = new Protomux(socket) - - if (fileOperation === 'upload') { - const upload = new UploadClient({ sourcePath, targetPath }, { node, socket, mux }) - upload.open() - } else { - const download = new DownloadClient({ sourcePath, targetPath }, { node, socket, mux }) - download.open() - } -} - -function errorAndExit (message) { - console.error('Error:', message) - process.exit(1) -} - -function parseRemotePath (str) { - const i = str.indexOf(':') - if (i === -1) return [null, null] - - const isName = str[0] === '@' - return [str.slice(isName ? 1 : 0, i), str.slice(i + 1)] // [host, path] -} + .option('-f ', 'filename of the seed key file') + .action(require('../lib/bin/copy.js')) diff --git a/bin/login.js b/bin/login.js index 0937cc3..7751b4c 100644 --- a/bin/login.js +++ b/bin/login.js @@ -1,11 +1,9 @@ -const path = require('path') const { createCommand } = require('commander') -const constants = require('../lib/constants.js') module.exports = createCommand('login') .description('connect to a P2P shell') .argument('', 'public key or name of the server') - .option('-f ', 'filename of the client seed key', path.join(constants.dir, 'id')) + .option('-f ', 'filename of the client seed key') .option('-L <[address:]port:host:hostport...>', 'local port forwarding') .option('--bootstrap ', 'custom dht nodes') .action(require('../lib/bin/login.js')) diff --git a/index.js b/index.js index 95f5d34..ef852e6 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ const Protomux = require('protomux') const HypercoreId = require('hypercore-id-encoding') const crypto = require('hypercore-crypto') const { ShellServer, ShellClient } = require('./lib/protocols/shell.js') +const Copy = require('./lib/protocols/copy.js') module.exports = class Hypershell { constructor (opts = {}) { @@ -20,6 +21,18 @@ module.exports = class Hypershell { mux.pair({ protocol: 'hypershell' }, function () { ShellServer.attach(mux) }) + + mux.pair({ protocol: 'hypershell-copy' }, function () { + Copy.attach(mux, { permissions: ['pack', 'extract'] }) + }) + + /* mux.pair({ protocol: 'hypershell-copy-upload' }, function () { + CopyUpload.attach(mux) + }) + + mux.pair({ protocol: 'hypershell-copy-download' }, function () { + CopyDownload.attach(mux) + }) */ } }) } @@ -34,6 +47,29 @@ module.exports = class Hypershell { }) } + tunnel () { + + } + + copy (publicKey, opts = {}) { + const client = new Client(this.dht, publicKey, opts) + const mux = Protomux.from(client.socket) + + return { upload, download, close } + + async function upload (source, destination) { + await Copy.upload(mux, source, destination) + } + + async function download (source, destination) { + await Copy.download(mux, source, destination) + } + + async function close () { + mux.destroy() + } + } + async destroy () { if (this._autoDestroy) { await this.dht.destroy() diff --git a/lib/bin/copy.js b/lib/bin/copy.js new file mode 100644 index 0000000..1ee6b25 --- /dev/null +++ b/lib/bin/copy.js @@ -0,0 +1,58 @@ +const fs = require('fs') +const path = require('path') +const constants = require('../constants.js') +const keygen = require('./keygen.js') +const getKnownPeer = require('../get-known-peer.js') +const fileToKeyPair = require('../file-to-keypair.js') +const Hypershell = require('../../index.js') + +const publicKeyExpr = /^([a-fA-F0-9]{64}|[ybndrfg8ejkmcpqxot1uwisza345h769]{52}):/i + +module.exports = async function copy (source, destination, opts = {}) { + const keyFilename = path.resolve(opts.f || path.join(constants.dir, 'id')) + + if (!fs.existsSync(keyFilename)) { + await keygen({ filename: keyFilename }) + } + + const direction = source[0] === '@' || publicKeyExpr.test(source) ? 'download' : 'upload' + const keyOrName = parseRemotePath(direction === 'download' ? source : destination)[0] + + source = parseRemotePath(source)[1] + destination = parseRemotePath(destination)[1] + + const serverPublicKey = await getKnownPeer(keyOrName, { verbose: true }) + + const hs = new Hypershell({ + bootstrap: opts.bootstrap + }) + + const transfer = hs.copy(serverPublicKey, { + keyPair: await fileToKeyPair(keyFilename) + }) + + try { + if (direction === 'upload') { + await transfer.upload(source, destination) + } else { + await transfer.download(source, destination) + } + } finally { + await transfer.close() + await hs.destroy() + } +} + +function parseRemotePath (str) { + const i = str.indexOf(':') + + if (i === -1) { + return [null, str] + } + + const at = str[0] === '@' ? 1 : 0 + const host = str.slice(at, i) + const path = str.slice(i + 1) + + return [host, path] +} diff --git a/lib/bin/keygen.js b/lib/bin/keygen.js index 4aad0c8..a05eec5 100644 --- a/lib/bin/keygen.js +++ b/lib/bin/keygen.js @@ -6,7 +6,6 @@ const constants = require('../constants.js') const question = require('../question.js') module.exports = async function keygen (opts = {}) { - console.log(opts) let { f: filename = path.join(constants.dir, 'id'), comment = opts.comment ? (' # ' + opts.comment) : '' diff --git a/lib/bin/login.js b/lib/bin/login.js index 8700f6f..06ee794 100644 --- a/lib/bin/login.js +++ b/lib/bin/login.js @@ -1,24 +1,26 @@ const fs = require('fs') +const path = require('path') +const constants = require('../constants.js') const keygen = require('./keygen.js') const Hypershell = require('../../index.js') const getKnownPeer = require('../get-known-peer.js') const fileToKeyPair = require('../file-to-keypair.js') -module.exports = async function login (serverPublicKey, opts = {}) { - console.log('login', opts) +module.exports = async function login (keyOrName, opts = {}) { + const keyFilename = path.resolve(opts.f || path.join(constants.dir, 'id')) - if (!fs.existsSync(opts.f)) { - await keygen({ filename: opts.f }) + if (!fs.existsSync(keyFilename)) { + await keygen({ filename: keyFilename }) } - const target = await getKnownPeer(serverPublicKey, { verbose: true }) + const serverPublicKey = await getKnownPeer(keyOrName, { verbose: true }) const hs = new Hypershell({ bootstrap: opts.bootstrap }) - const shell = hs.login(target, { - keyPair: await fileToKeyPair(opts.f), + const shell = hs.login(serverPublicKey, { + keyPair: await fileToKeyPair(keyFilename), rawArgs: this.rawArgs, stdin: process.stdin, stdout: process.stdout, diff --git a/lib/bin/server.js b/lib/bin/server.js index e8f9ddf..f2141aa 100644 --- a/lib/bin/server.js +++ b/lib/bin/server.js @@ -12,8 +12,6 @@ const fileToKeyPair = require('../file-to-keypair.js') const PROTOCOLS = ['shell', 'upload', 'download', 'tunnel'] module.exports = async function server (opts = {}) { - console.log('server', opts) - const keyFilename = path.resolve(opts.f || path.join(constants.dir, 'id')) const firewallFilename = path.resolve(opts.firewall || path.join(constants.dir, 'authorized_peers')) const firewallEnabled = !opts.disableFirewall diff --git a/lib/protocols/copy.js b/lib/protocols/copy.js new file mode 100644 index 0000000..82e3730 --- /dev/null +++ b/lib/protocols/copy.js @@ -0,0 +1,181 @@ +const fs = require('fs') +const os = require('os') +const path = require('path') +const tar = require('tar-fs') +const c = require('compact-encoding') +const m = require('./messages.js') + +const EMPTY = Buffer.alloc(0) + +module.exports = class Copy { + constructor (mux, opts = {}) { + this.channel = mux.createChannel({ + protocol: 'hypershell-copy', + unique: false, + // handshake: m.handshakeCopy, + onopen: this.onopen.bind(this), + onclose: this.onclose.bind(this), + messages: [ + { encoding: m.copyHeader, onmessage: this.onWireHeader.bind(this) }, + { encoding: c.raw, onmessage: this.onWireData.bind(this) }, + { encoding: m.error, onmessage: this.onWireError.bind(this) } + ] + }) + + this.wireHeader = this.channel.messages[0] + this.wireData = this.channel.messages[1] + this.wireError = this.channel.messages[2] + + this.permissions = opts.permissions || [] + this.tar = null + this.error = null + + this._onerror = opts.onerror || null + this._dst = null + } + + static attach (mux, opts) { + const copy = new this(mux, opts) + + copy.channel.open() + } + + static async upload (mux, source, destination) { + const copy = new this(mux) + + copy.upload(source, destination) + + await copy.channel.fullyClosed() + + if (copy.error) { + throw makeError(copy.error) + } + } + + static async download (mux, source, destination) { + const copy = new this(mux, { permissions: ['extract'] }) + + copy.download(source, destination) + + await copy.channel.fullyClosed() + + if (copy.error) { + throw makeError(copy.error) + } + } + + upload (source, destination) { + this.channel.open() + + this._pack(source, destination) + } + + download (source, destination) { + this.channel.open() + + // The server side should not know either control client's destination + this._dst = destination + this.wireHeader.send({ pack: source, destination: null }) + } + + onopen (h) {} + + onclose () { + if (this.tar) this.tar.destroy() + } + + onWireHeader (data, c) { + const action = data.pack ? 'pack' : 'extract' + + if (!this.permissions.includes(action)) { + this._exit(new Error('Action is not allowed: ' + action)) + return + } + + if (data.pack) { + this._pack(data.pack, data.destination) + } else if (this._dst || data.extract) { + this._extract(this._dst || data.extract, data.sourceIsDirectory) + } + } + + onWireData (data, c) { + if (!this.tar) return + + if (data.length) this.tar.write(data) + else this.tar.end() + } + + onWireError (data, c) { + if (this._onerror) this._onerror(data) + + this.error = data + + c.close() + } + + _pack (source, destination) { + source = path.resolve(resolveHomedir(source)) + + try { + const st = fs.lstatSync(source) + const sourceIsDirectory = st.isDirectory() + + this.wireHeader.send({ extract: destination, sourceIsDirectory }) + } catch (err) { + this._exit(err, source) + return + } + + this.tar = tar.pack(source) + this.tar.on('data', chunk => this.wireData.send(chunk)) + this.tar.on('end', () => this.wireData.send(EMPTY)) + this.tar.on('error', err => this._exit(err)) + } + + _extract (destination, isDirectory) { + destination = path.resolve(resolveHomedir(destination)) + + const dir = isDirectory ? destination : path.dirname(destination) + const options = { + readable: true, + writable: true, + map: header => { + if (!isDirectory) { + header.name = path.basename(destination) + } + + return header + } + } + + this.tar = tar.extract(dir, options) + this.tar.on('finish', () => this.channel.close()) + this.tar.on('error', err => this._exit(err)) + } + + _exit (err, extra) { + this.error = err + + if (this._onerror) this._onerror(err, extra) + else this.wireError.send(err) + + this.channel.close() + } +} + +function resolveHomedir (str) { + if (!str) return str + if (str === '~') return os.homedir() + if (str.slice(0, 2) !== '~/') return str + return path.join(os.homedir(), str.slice(2)) +} + +function makeError (error) { + const err = new Error(error.message) + + if (error.code) err.code = error.code + if (error.path) err.path = error.path + + return err +} diff --git a/lib/protocols/messages.js b/lib/protocols/messages.js index 3e5a3b4..c389387 100644 --- a/lib/protocols/messages.js +++ b/lib/protocols/messages.js @@ -42,30 +42,25 @@ const handshakeUpload = { } } -const handshakeDownload = { - preencode (state, u) { - c.string.preencode(state, u.source || '') +const copyHeader = { + preencode (state, h) { + c.string.preencode(state, h.pack || '') + c.string.preencode(state, h.extract || '') + c.string.preencode(state, h.destination || '') + c.bool.preencode(state, h.sourceIsDirectory || false) }, - encode (state, u) { - c.string.encode(state, u.source || '') + encode (state, h) { + c.string.encode(state, h.pack || '') + c.string.encode(state, h.extract || '') + c.string.encode(state, h.destination || '') + c.bool.encode(state, h.sourceIsDirectory || false) }, decode (state) { return { - source: c.string.decode(state) - } - } -} - -const downloadHeader = { - preencode (state, d) { - c.bool.preencode(state, d.isDirectory) - }, - encode (state, d) { - c.bool.encode(state, d.isDirectory) - }, - decode (state) { - return { - isDirectory: c.bool.decode(state) + pack: c.string.decode(state), + extract: c.string.decode(state), + destination: c.string.decode(state), + sourceIsDirectory: c.bool.decode(state) } } } @@ -110,8 +105,7 @@ const resize = { module.exports = { handshakeSpawn, handshakeUpload, - handshakeDownload, - downloadHeader, + copyHeader, error, resize } diff --git a/lib/protocols/tunnel.js b/lib/protocols/tunnel.js new file mode 100644 index 0000000..8e40d4e --- /dev/null +++ b/lib/protocols/tunnel.js @@ -0,0 +1,250 @@ +const net = require('net') +const c = require('compact-encoding') +const pump = require('pump') +const DHT = require('hyperdht') +const Protomux = require('protomux') +const SecretStream = require('@hyperswarm/secret-stream') +const ReadyResource = require('ready-resource') +const safetyCatch = require('safety-catch') + +class LocalTunnelServer { + constructor ({ node, socket, mux, options }) { + this.dht = node + this.socket = socket + + this.channel = mux.createChannel({ + protocol: 'hypershell-tunnel-local', + id: null, + handshake: c.json, + onopen: this.onopen.bind(this), + onclose: this.onclose.bind(this), + messages: [ + { encoding: c.json, onmessage: this.onstreamid.bind(this) } + ] + }) + + this.options = options + + this.streams = new Map() + this.config = null + } + + open () { + this.channel.open({}) + } + + onopen (handshake) { + const isHostAllowed = LocalTunnelServer.firewallHosts(this.options.tunnelHost, handshake.host) + const isPortAllowed = LocalTunnelServer.firewallPorts(this.options.tunnelPort, handshake.port) + + if (!isHostAllowed || !isPortAllowed) { + this.channel.close() + return + } + + this.config = handshake + } + + onclose () { + for (const [, stream] of this.streams) { + stream.destroy() + } + } + + onstreamid (data, c) { + const { clientId } = data + + const rawStream = this.dht.createRawStream() + this.streams.set(rawStream.id, rawStream) + rawStream.on('close', () => this.streams.delete(rawStream.id)) + rawStream.on('error', safetyCatch) + + c.messages[0].send({ clientId, serverId: rawStream.id }) + + DHT.connectRawStream(this.socket, rawStream, clientId) + const secretStream = new SecretStream(true, rawStream) + secretStream.on('error', safetyCatch) + + secretStream.setKeepAlive(5000) + + const remoteSocket = net.connect(this.config.port, this.config.host) + rawStream.userData = { remoteSocket, secretStream } + + pump(secretStream, remoteSocket, secretStream) + } + + static firewallHosts (hosts, target) { + if (!hosts) return true + + for (const host of hosts) { + // + support for CIDR ranges? + if (host === target) return true + } + + return false + } + + static firewallPorts (ports, target) { + if (!ports) return true + + for (const port of ports) { + const isRange = port.indexOf('-') > -1 + let list = null + + if (isRange) { + const [start, end] = port.split('-', 2).map(Number) + const length = end - start + 1 + + list = Array.from({ length }, (_, i) => start + i) + } else { + list = [Number(port)] + } + + for (const number of list) { + if (number === target) return true + } + } + + return false + } +} + +class LocalTunnelClient extends ReadyResource { + constructor (config, { node, keyPair, serverPublicKey }) { + super() + + this.dht = node + + this.keyPair = keyPair + this.serverPublicKey = serverPublicKey + + this.config = LocalTunnelClient.parse(config) // + defaults + + this.streams = new Map() + this.server = net.createServer(this.onconnection.bind(this)) // + option for udp + + this.ready().catch(safetyCatch) + } + + async _open () { + this.server.listen(this.config.local.port, this.config.local.host) + + await waitForServer(this.server) + } + + _close () { + this.server.close() + + if (this.mux) this.mux.destroy() + } + + _createMux () { + if (this.mux && !this.mux.stream.destroying) return + + // + reusableSocket for when having several -L tunnels? + const socket = this.dht.connect(this.serverPublicKey, { keyPair: this.keyPair }) + + socket.setKeepAlive(5000) + + this.mux = new Protomux(socket) + } + + _createChannel () { + if (this.mux.opened({ protocol: 'hypershell-tunnel-local', id: null })) return + + const channel = this.mux.createChannel({ + protocol: 'hypershell-tunnel-local', + id: null, + handshake: c.json, + messages: [ + { encoding: c.json, onmessage: this.onstreamid.bind(this) } + ], + onopen: this.onopen.bind(this), + onclose: this.onclose.bind(this) + }) + + if (channel === null) return + + this.channel = channel + this.channel.open(this.config.remote) + } + + onopen () { + // No-op + } + + onclose () { + for (const [, stream] of this.streams) { + stream.destroy() + } + } + + onconnection (localSocket) { + this._createMux() + this._createChannel() + + const rawStream = this.dht.createRawStream() + this.streams.set(rawStream.id, rawStream) + rawStream.on('close', () => this.streams.delete(rawStream.id)) + rawStream.on('error', safetyCatch) + + rawStream.userData = { localSocket } + rawStream.on('close', () => localSocket.destroy()) + localSocket.on('error', safetyCatch) + + this.channel.messages[0].send({ clientId: rawStream.id, serverId: 0 }) + } + + onstreamid (data, channel) { + const { clientId, serverId } = data + + const rawStream = this.streams.get(clientId) + if (!rawStream) throw new Error('Stream not found: ' + clientId) + const { localSocket } = rawStream.userData + + DHT.connectRawStream(this.mux.stream, rawStream, serverId) + const secretStream = new SecretStream(false, rawStream) + secretStream.on('error', safetyCatch) + + secretStream.setKeepAlive(5000) + + rawStream.userData.secretStream = secretStream + + pump(localSocket, secretStream, localSocket) + } + + static parse (config) { + const match = config.match(/(?:(.*):)?([\d]+):(?:(.*):)?([\d]+)/i) + + // + should return errors + if (!match[2]) errorAndExit('local port is required') + if (!match[3]) errorAndExit('remote host is required') + if (!match[4]) errorAndExit('remote port is required') + + const local = { host: match[1] || '0.0.0.0', port: Number(match[2]) } + const remote = { host: match[3], port: Number(match[4]) } + + return { local, remote } + } +} + +function waitForServer (server) { + return new Promise((resolve, reject) => { + server.on('listening', done) + server.on('error', done) + // if (server.listening) done() + + function done (error) { + server.removeListener('listening', done) + server.removeListener('error', done) + error ? reject(error) : resolve() + } + }) +} + +module.exports = { LocalTunnelServer, LocalTunnelClient } + +function errorAndExit (message) { + console.error('Error:', message) + process.exit(1) +} diff --git a/package.json b/package.json index 08511f0..518f5b8 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "protomux-rpc": "^1.6.0", "pump": "^3.0.2", "read-file-live": "^1.0.1", + "tar-fs": "^3.0.6", "tiny-configs": "^1.1.0", "tt-native": "^1.1.1" } diff --git a/test/lib.js b/test/lib.js index e9109b0..dadd7f5 100644 --- a/test/lib.js +++ b/test/lib.js @@ -1,4 +1,7 @@ +const fs = require('fs') +const path = require('path') const test = require('brittle') +const tmp = require('like-tmp') const crypto = require('hypercore-crypto') const { getStreamError } = require('streamx') const createTestnet = require('hyperdht/testnet') @@ -60,6 +63,59 @@ test('server is closed first', async function (t) { t.is(err.code, 'ECONNRESET') }) +test.skip('basic tunnel', async function (t) { + t.plan(2) + + const hs = await createHypershell(t) + const server = await createServer(hs) + const tunnel = hs.tunnel(server.publicKey) + + await tunnel.local('127.0.0.1:8080', '127.0.0.1:80') + + await tunnel.close() + await server.close() + await hs.destroy() +}) + +test('basic copy', async function (t) { + t.plan(6) + + const hs = await createHypershell(t) + const server = await createServer(hs) + const transfer = hs.copy(server.publicKey) + + const dir = await tmp(t) + const msg = Buffer.from('Hello World!') + + await fs.promises.writeFile(path.join(dir, 'file.txt'), msg) + + await transfer.upload(path.join(dir, 'file.txt'), path.join(dir, 'uploaded.txt')) + t.alike(await fs.promises.readFile(path.join(dir, 'uploaded.txt')), msg) + + await transfer.download(path.join(dir, 'uploaded.txt'), path.join(dir, 'downloaded.txt')) + t.alike(await fs.promises.readFile(path.join(dir, 'downloaded.txt')), msg) + + try { + await transfer.upload(path.join(dir, 'not-exists.txt'), path.join(dir, 'uploaded.txt')) + t.fail() + } catch (err) { + t.is(err.code, 'ENOENT') + t.ok(err.path) + } + + try { + await transfer.download(path.join(dir, 'not-exists.txt'), path.join(dir, 'downloaded.txt')) + t.fail() + } catch (err) { + t.is(err.code, 'ENOENT') + t.ok(err.path) + } + + await transfer.close() + await server.close() + await hs.destroy() +}) + async function createHypershell (t) { const { bootstrap } = await createTestnet(3, t.teardown) From 0a0f8c8e4ca74ca06bca84f628deda6fb8ebd07c Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Wed, 16 Oct 2024 19:43:48 -0300 Subject: [PATCH 03/21] Tunnels --- README.md | 31 +++- bin.js | 1 + bin/copy.js | 1 + bin/login.js | 1 - bin/server.js | 3 +- index.js | 47 +++-- lib/bin/server.js | 13 +- lib/next-id.js | 11 ++ lib/protocols/tunnel.js | 372 +++++++++++++++++++++++----------------- package.json | 1 + test/lib.js | 194 ++++++++++++++++++--- 11 files changed, 464 insertions(+), 211 deletions(-) create mode 100644 lib/next-id.js diff --git a/README.md b/README.md index 1924f01..a3820ec 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,10 @@ hypershell keygen [-f keyfile] [-c comment] hypershell server [-f keyfile] [--firewall filename] [--disable-firewall] [--protocol name] # Connect to a P2P shell -hypershell login [-f keyfile] +hypershell login [-f keyfile] # Local tunnel that forwards to remote host -hypershell tunnel -L [address:]port:host:hostport +hypershell tunnel -L [address:]port:host:hostport # Copy files (download and upload) hypershell copy <[@host:]source> <[@host:]target> [-f keyfile] @@ -34,11 +34,22 @@ It can also be imported as a library: ```js const Hypershell = require('hypershell') -const keyPair = Hypershell.keygen({ filename, comment }) +const hs = new Hypershell() -const closeServer = await Hypershell.server({ firewall }) -const closeServer = await Hypershell.login({ firewall }) +const server = hs.createServer() +await server.listen() +const keyPair = crypto.keyPair() + +server.firewall = [keyPair.publicKey] + +const shell = hs.login(server.publicKey, { keyPair }) + +shell.stdin.write('echo "Hello World!"\n') + +await shell.close() +await server.close() +await hs.destroy() ``` ## First steps @@ -112,6 +123,12 @@ In this example, creates a local tunnel at `127.0.0.1:2020` (where you can conne that later gets forwarded to a remote server which it connects to `127.0.0.1:3000`: ```bash hypershell remote_peer -L 127.0.0.1:2020:127.0.0.1:3000 + +# Local 8080 -> Remote 1080 +hypershell tunnel -L 8080:127.0.0.1:1080:127.0.0.1 + +# Remote 80 -> Local 3000 +hypershell tunnel -R 80:127.0.0.1:3000:127.0.0.1 ``` Instead of `remote_peer` you can use the server public key as well. @@ -131,13 +148,13 @@ Let's say you have a local project like a React app at `http://127.0.0.1:3000/`, you can create a restricted server to safely share this unique port like so: ```bash -hypershell-server --protocol tunnel --tunnel-host 127.0.0.1 --tunnel-port 3000 +hypershell server --protocol tunnel --tunnel 127.0.0.1:3000 ``` Or if you want to allow multiple hosts, port range, etc: ```bash -hypershell-server --protocol tunnel --tunnel-host 127.0.0.1 --tunnel-host 192.168.0.25 --tunnel-port 1080 --tunnel-port 3000 --tunnel-port 4100-4200 +hypershell server --protocol tunnel --tunnel 127.0.0.1:4100-4200 --tunnel 192.168.0.25:1080 ``` Clients trying to use any different hosts/ports are automatically disconnected. diff --git a/bin.js b/bin.js index c620258..4ffc9ac 100755 --- a/bin.js +++ b/bin.js @@ -11,6 +11,7 @@ const main = program .addCommand(require('./bin/server.js')) .addCommand(require('./bin/login.js')) .addCommand(require('./bin/copy.js')) + .addCommand(require('./bin/tunnel.js')) main.parseAsync().catch(err => { safetyCatch(err) diff --git a/bin/copy.js b/bin/copy.js index b83a419..82daef1 100644 --- a/bin/copy.js +++ b/bin/copy.js @@ -5,4 +5,5 @@ module.exports = createCommand('copy') .argument('', 'Source') .argument('', 'Target') .option('-f ', 'filename of the seed key file') + .option('--bootstrap ', 'custom dht nodes') .action(require('../lib/bin/copy.js')) diff --git a/bin/login.js b/bin/login.js index 7751b4c..33a1886 100644 --- a/bin/login.js +++ b/bin/login.js @@ -4,6 +4,5 @@ module.exports = createCommand('login') .description('connect to a P2P shell') .argument('', 'public key or name of the server') .option('-f ', 'filename of the client seed key') - .option('-L <[address:]port:host:hostport...>', 'local port forwarding') .option('--bootstrap ', 'custom dht nodes') .action(require('../lib/bin/login.js')) diff --git a/bin/server.js b/bin/server.js index 1576379..408297d 100644 --- a/bin/server.js +++ b/bin/server.js @@ -6,7 +6,6 @@ module.exports = createCommand('server') .option('--firewall ', 'list of allowed public keys') .option('--disable-firewall', 'allow anyone to connect', false) .option('--protocol ', 'list of allowed protocols') - .option('--tunnel-host ', 'restrict tunneling to a limited set of hosts') - .option('--tunnel-port ', 'restrict tunneling to a limited set of ports') + .option('--tunnel ', 'restrict tunneling to a limited set of addresses') .option('--bootstrap ', 'custom dht nodes') .action(require('../lib/bin/server.js')) diff --git a/index.js b/index.js index ef852e6..e10eb6b 100644 --- a/index.js +++ b/index.js @@ -4,6 +4,7 @@ const HypercoreId = require('hypercore-id-encoding') const crypto = require('hypercore-crypto') const { ShellServer, ShellClient } = require('./lib/protocols/shell.js') const Copy = require('./lib/protocols/copy.js') +const Tunnel = require('./lib/protocols/tunnel.js') module.exports = class Hypershell { constructor (opts = {}) { @@ -18,21 +19,28 @@ module.exports = class Hypershell { onsocket: function (socket) { const mux = Protomux.from(socket) - mux.pair({ protocol: 'hypershell' }, function () { - ShellServer.attach(mux) - }) - - mux.pair({ protocol: 'hypershell-copy' }, function () { - Copy.attach(mux, { permissions: ['pack', 'extract'] }) - }) - - /* mux.pair({ protocol: 'hypershell-copy-upload' }, function () { - CopyUpload.attach(mux) - }) - - mux.pair({ protocol: 'hypershell-copy-download' }, function () { - CopyDownload.attach(mux) - }) */ + if (this.protocols.includes('shell')) { + mux.pair({ protocol: 'hypershell' }, () => { + ShellServer.attach(mux) + }) + } + + if (this.protocols.includes('copy')) { + mux.pair({ protocol: 'hypershell-copy' }, () => { + Copy.attach(mux, { permissions: ['pack', 'extract'] }) + }) + } + + if (this.protocols.includes('tunnel')) { + mux.pair({ protocol: 'hypershell-tunnel' }, () => { + const tunnel = new Tunnel(this.dht, null, { + mux, + allow: opts.tunnel?.allow + }) + + tunnel._createChannel() + }) + } } }) } @@ -47,10 +55,6 @@ module.exports = class Hypershell { }) } - tunnel () { - - } - copy (publicKey, opts = {}) { const client = new Client(this.dht, publicKey, opts) const mux = Protomux.from(client.socket) @@ -70,6 +74,10 @@ module.exports = class Hypershell { } } + tunnel (publicKey, opts = {}) { + return new Tunnel(this.dht, publicKey, opts) + } + async destroy () { if (this._autoDestroy) { await this.dht.destroy() @@ -83,6 +91,7 @@ class Server { this.keyPair = opts.keyPair || crypto.keyPair(opts.seed) this.firewall = opts.firewall || opts.firewall === null ? opts.firewall : [] this.verbose = !!opts.verbose + this.protocols = opts.protocols || ['shell', 'copy', 'tunnel'] this._server = this.dht.createServer({ firewall: this._onFirewall.bind(this) diff --git a/lib/bin/server.js b/lib/bin/server.js index f2141aa..28cf35d 100644 --- a/lib/bin/server.js +++ b/lib/bin/server.js @@ -9,13 +9,10 @@ const keygen = require('./keygen.js') const Hypershell = require('../../index.js') const fileToKeyPair = require('../file-to-keypair.js') -const PROTOCOLS = ['shell', 'upload', 'download', 'tunnel'] - module.exports = async function server (opts = {}) { - const keyFilename = path.resolve(opts.f || path.join(constants.dir, 'id')) + const keyFilename = path.resolve(opts.f || path.join(constants.dir, 'id_server')) const firewallFilename = path.resolve(opts.firewall || path.join(constants.dir, 'authorized_peers')) const firewallEnabled = !opts.disableFirewall - const protocols = opts.protocol || PROTOCOLS if (!fs.existsSync(keyFilename)) { await keygen({ filename: keyFilename }) @@ -33,6 +30,8 @@ module.exports = async function server (opts = {}) { }) const server = hs.createServer({ + protocols: opts.protocol, + tunnel: { allow: opts.tunnel }, keyPair: await fileToKeyPair(keyFilename), verbose: true }) @@ -49,11 +48,11 @@ module.exports = async function server (opts = {}) { await server.listen() - if (protocols === PROTOCOLS) { + if (server.protocols.includes('shell')) { console.log('To connect to this shell, on another computer run:') - console.log('hypershell ' + HypercoreId.encode(server.publicKey)) + console.log('hypershell login ' + HypercoreId.encode(server.publicKey)) } else { - console.log('Running server with restricted protocols') + console.log('Running server with restricted protocols:', server.protocols.join(', ')) console.log('Server key: ' + HypercoreId.encode(server.publicKey)) } console.log() diff --git a/lib/next-id.js b/lib/next-id.js new file mode 100644 index 0000000..7a03074 --- /dev/null +++ b/lib/next-id.js @@ -0,0 +1,11 @@ +module.exports = function nextId () { + let id = 1 + + return function () { + if (id === 0xffffffff) { + id = 1 + } + + return id++ + } +} diff --git a/lib/protocols/tunnel.js b/lib/protocols/tunnel.js index 8e40d4e..763e34b 100644 --- a/lib/protocols/tunnel.js +++ b/lib/protocols/tunnel.js @@ -4,247 +4,309 @@ const pump = require('pump') const DHT = require('hyperdht') const Protomux = require('protomux') const SecretStream = require('@hyperswarm/secret-stream') -const ReadyResource = require('ready-resource') +const listen = require('listen-async') const safetyCatch = require('safety-catch') +const nextId = require('../next-id') -class LocalTunnelServer { - constructor ({ node, socket, mux, options }) { - this.dht = node - this.socket = socket +module.exports = class Tunnel { + constructor (dht, publicKey, opts = {}) { + this.dht = dht + this.publicKey = publicKey - this.channel = mux.createChannel({ - protocol: 'hypershell-tunnel-local', - id: null, + this.keyPair = opts.keyPair || null + this.allow = opts.allow || null + + this.mux = opts.mux || null + this.channel = null + + this.wireCommand = null + this.wireStream = null + this.wirePump = null + + this.streams = new Map() + this.proxies = new Set() + + this.nextId = nextId() + } + + _connect () { + if (this.mux && !this.mux.stream.destroying) { + return + } + + const socket = this.dht.connect(this.publicKey, { + keyPair: this.keyPair, + reusableSocket: true + }) + + socket.setKeepAlive(5000) + + this.mux = Protomux.from(socket) + } + + _createChannel () { + if (this.mux.opened({ protocol: 'hypershell-tunnel' })) { + return + } + + const channel = this.mux.createChannel({ + protocol: 'hypershell-tunnel', handshake: c.json, onopen: this.onopen.bind(this), onclose: this.onclose.bind(this), messages: [ - { encoding: c.json, onmessage: this.onstreamid.bind(this) } + { encoding: c.json, onmessage: this.onMessage.bind(this) }, + { encoding: c.json, onmessage: this.onWireServer.bind(this) }, + { encoding: c.json, onmessage: this.onWireConnect.bind(this) }, + { encoding: c.json, onmessage: this.onWirePump.bind(this) } ] }) - this.options = options + if (channel === null) { + return + } - this.streams = new Map() - this.config = null - } + this.channel = channel + + this.wireMessage = this.channel.messages[0] + this.wireServer = this.channel.messages[1] + this.wireConnect = this.channel.messages[2] + this.wirePump = this.channel.messages[3] - open () { this.channel.open({}) } - onopen (handshake) { - const isHostAllowed = LocalTunnelServer.firewallHosts(this.options.tunnelHost, handshake.host) - const isPortAllowed = LocalTunnelServer.firewallPorts(this.options.tunnelPort, handshake.port) + onopen (h) {} - if (!isHostAllowed || !isPortAllowed) { - this.channel.close() - return - } - - this.config = handshake - } + async onclose () { + // TODO - onclose () { for (const [, stream] of this.streams) { stream.destroy() } - } - onstreamid (data, c) { - const { clientId } = data + for (const proxy of this.proxies) { + await proxy.close() + } - const rawStream = this.dht.createRawStream() - this.streams.set(rawStream.id, rawStream) - rawStream.on('close', () => this.streams.delete(rawStream.id)) - rawStream.on('error', safetyCatch) + this.streams.clear() + this.proxies.clear() + } - c.messages[0].send({ clientId, serverId: rawStream.id }) + async local (localAddress, remoteAddress) { + let fwd = parseForwardFormat(localAddress + (remoteAddress ? ':' + remoteAddress : '')) - DHT.connectRawStream(this.socket, rawStream, clientId) - const secretStream = new SecretStream(true, rawStream) - secretStream.on('error', safetyCatch) - - secretStream.setKeepAlive(5000) + const server = net.createServer(localSocket => { + this._onLocalConnection(fwd, localSocket) + }) - const remoteSocket = net.connect(this.config.port, this.config.host) - rawStream.userData = { remoteSocket, secretStream } + await listen(server, fwd.local.port, fwd.local.host) - pump(secretStream, remoteSocket, secretStream) + return { + forwardTo: function (remoteAddress) { + fwd = parseForwardFormat(localAddress + ':' + remoteAddress) + }, + close: function () { + // TODO: Force close connections + return new Promise(resolve => server.close(resolve)) + } + } } - static firewallHosts (hosts, target) { - if (!hosts) return true + async remote (remoteAddress, localAddress) { + const fwd = parseForwardFormat((localAddress ? localAddress + ':' : '') + remoteAddress) - for (const host of hosts) { - // + support for CIDR ranges? - if (host === target) return true - } + // TODO: This is to get a new isolated channel easly for now + // So that if connection is lost then the other side can drop its resources + // Client should handle reconnection e.g. re-execute remote() + const instance = new Tunnel(this.dht, this.publicKey, { + keyPair: this.keyPair, + allow: [fwd.local.host + ':' + fwd.local.port] + }) - return false - } + instance._connect() + instance._createChannel() - static firewallPorts (ports, target) { - if (!ports) return true + // Only one remote server per channel + instance.wireServer.send({ + port: fwd.remote.port, + host: fwd.remote.host, + connect: fwd.local + }) - for (const port of ports) { - const isRange = port.indexOf('-') > -1 - let list = null + await instance.channel.fullyOpened() - if (isRange) { - const [start, end] = port.split('-', 2).map(Number) - const length = end - start + 1 + console.log('fullyOpened') - list = Array.from({ length }, (_, i) => start + i) - } else { - list = [Number(port)] - } + return { + close: async function () { + instance.mux.destroy() - for (const number of list) { - if (number === target) return true + await instance.channel.fullyClosed() } } - - return false } -} -class LocalTunnelClient extends ReadyResource { - constructor (config, { node, keyPair, serverPublicKey }) { - super() + async close () {} - this.dht = node + _onLocalConnection (fwd, localSocket) { + this._connect() + this._createChannel() - this.keyPair = keyPair - this.serverPublicKey = serverPublicKey + const rawStream = this._createRawStream() - this.config = LocalTunnelClient.parse(config) // + defaults + rawStream.userData = { localSocket } - this.streams = new Map() - this.server = net.createServer(this.onconnection.bind(this)) // + option for udp + rawStream.on('close', () => localSocket.destroy()) - this.ready().catch(safetyCatch) - } + localSocket.on('error', safetyCatch) - async _open () { - this.server.listen(this.config.local.port, this.config.local.host) + this.wireConnect.send({ + clientId: rawStream.id, + connect: fwd.remote + }) + } - await waitForServer(this.server) + async onMessage (data, c) { + if (data.state === 'WIRE_SERVER_READY') { + this.resolve + } } - _close () { - this.server.close() + async onWireServer (data, c) { + const { port, host, connect } = data - if (this.mux) this.mux.destroy() - } + console.log('onWireServer A') - _createMux () { - if (this.mux && !this.mux.stream.destroying) return + const proxy = await this.local(port + ':' + host, connect.port + ':' + connect.host) - // + reusableSocket for when having several -L tunnels? - const socket = this.dht.connect(this.serverPublicKey, { keyPair: this.keyPair }) + // TODO: Check if channel got closed - socket.setKeepAlive(5000) + this.proxies.add(proxy) + + await new Promise(resolve => setTimeout(resolve, 500)) - this.mux = new Protomux(socket) + this.wireMessage.send({ state: 'WIRE_SERVER_READY' }) + + console.log('onWireServer B') } - _createChannel () { - if (this.mux.opened({ protocol: 'hypershell-tunnel-local', id: null })) return + onWireConnect (data, c) { + const { clientId, connect } = data - const channel = this.mux.createChannel({ - protocol: 'hypershell-tunnel-local', - id: null, - handshake: c.json, - messages: [ - { encoding: c.json, onmessage: this.onstreamid.bind(this) } - ], - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this) - }) + if (firewallTunnel(this.allow, connect)) { + c.close() + return + } - if (channel === null) return + const rawStream = this._createRawStream() + const secretStream = this._connectRawStream(rawStream, clientId, true) + const remoteSocket = net.connect(connect.port, connect.host) - this.channel = channel - this.channel.open(this.config.remote) - } + rawStream.userData = { secretStream, remoteSocket } + + pump(secretStream, remoteSocket, secretStream) - onopen () { - // No-op + this.wirePump.send({ + clientId, + serverId: rawStream.id + }) } - onclose () { - for (const [, stream] of this.streams) { - stream.destroy() + onWirePump (data, c) { + const { clientId, serverId } = data + + const rawStream = this.streams.get(clientId) + + if (!rawStream) { + throw new Error('Stream not found: ' + clientId) } - } - onconnection (localSocket) { - this._createMux() - this._createChannel() + const { localSocket } = rawStream.userData + + const secretStream = this._connectRawStream(rawStream, serverId, false) + rawStream.userData.secretStream = secretStream + + pump(localSocket, secretStream, localSocket) + } + + _createRawStream () { const rawStream = this.dht.createRawStream() - this.streams.set(rawStream.id, rawStream) + rawStream.on('close', () => this.streams.delete(rawStream.id)) rawStream.on('error', safetyCatch) - rawStream.userData = { localSocket } - rawStream.on('close', () => localSocket.destroy()) - localSocket.on('error', safetyCatch) + this.streams.set(rawStream.id, rawStream) - this.channel.messages[0].send({ clientId: rawStream.id, serverId: 0 }) + return rawStream } - onstreamid (data, channel) { - const { clientId, serverId } = data + _connectRawStream (rawStream, id, isInitiator) { + DHT.connectRawStream(this.mux.stream, rawStream, id) - const rawStream = this.streams.get(clientId) - if (!rawStream) throw new Error('Stream not found: ' + clientId) - const { localSocket } = rawStream.userData + const secretStream = new SecretStream(isInitiator, rawStream) - DHT.connectRawStream(this.mux.stream, rawStream, serverId) - const secretStream = new SecretStream(false, rawStream) secretStream.on('error', safetyCatch) secretStream.setKeepAlive(5000) - rawStream.userData.secretStream = secretStream + return secretStream + } +} - pump(localSocket, secretStream, localSocket) +function parseForwardFormat (value) { + const local = { port: null, host: null } + const remote = { port: null, host: null } + + for (const part of value.split(':')) { + const isNumber = !isNaN(part) + + if (isNumber) { + if (!local.port) local.port = parseInt(part, 10) + else if (!remote.port) remote.port = parseInt(part, 10) + else throw new Error('Invalid port format') + } else { + if (remote.port) remote.host = part + else if (local.port) local.host = part + else throw new Error('Invalid address format') + } } - static parse (config) { - const match = config.match(/(?:(.*):)?([\d]+):(?:(.*):)?([\d]+)/i) + return { local, remote } +} - // + should return errors - if (!match[2]) errorAndExit('local port is required') - if (!match[3]) errorAndExit('remote host is required') - if (!match[4]) errorAndExit('remote port is required') +function firewallTunnel (addresses, target) { + if (!addresses) { + return false + } - const local = { host: match[1] || '0.0.0.0', port: Number(match[2]) } - const remote = { host: match[3], port: Number(match[4]) } + for (const address of addresses) { + const [host, port] = address.split(':') - return { local, remote } - } -} + if (target.host !== host) { + continue + } + + if (port) { + const ports = port.split('-') -function waitForServer (server) { - return new Promise((resolve, reject) => { - server.on('listening', done) - server.on('error', done) - // if (server.listening) done() + if (ports.length === 1) { + if (target.port !== parseInt(port, 10)) { + continue + } + } else { + const from = parseInt(ports[0], 10) + const to = parseInt(ports[1], 10) - function done (error) { - server.removeListener('listening', done) - server.removeListener('error', done) - error ? reject(error) : resolve() + if (target.port < from || target.port > to) { + continue + } + } } - }) -} -module.exports = { LocalTunnelServer, LocalTunnelClient } + return false + } -function errorAndExit (message) { - console.error('Error:', message) - process.exit(1) + return true } diff --git a/package.json b/package.json index 518f5b8..130fde8 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "hypercore-crypto": "^3.4.2", "hypercore-id-encoding": "^1.3.0", "hyperdht": "^6.19.0", + "listen-async": "^1.0.0", "protomux": "^3.10.0", "protomux-rpc": "^1.6.0", "pump": "^3.0.2", diff --git a/test/lib.js b/test/lib.js index dadd7f5..720110f 100644 --- a/test/lib.js +++ b/test/lib.js @@ -1,10 +1,12 @@ const fs = require('fs') const path = require('path') +const net = require('net') const test = require('brittle') const tmp = require('like-tmp') const crypto = require('hypercore-crypto') const { getStreamError } = require('streamx') const createTestnet = require('hyperdht/testnet') +const listen = require('listen-async') const Hypershell = require('../index.js') test('basic shell', async function (t) { @@ -49,7 +51,10 @@ test('basic shell', async function (t) { test('server is closed first', async function (t) { const hs = await createHypershell(t) - const server = await createServer(hs) + + const server = hs.createServer({ firewall: null }) + await server.listen() + const shell = hs.login(server.publicKey) await shell.ready() @@ -63,25 +68,14 @@ test('server is closed first', async function (t) { t.is(err.code, 'ECONNRESET') }) -test.skip('basic tunnel', async function (t) { - t.plan(2) - - const hs = await createHypershell(t) - const server = await createServer(hs) - const tunnel = hs.tunnel(server.publicKey) - - await tunnel.local('127.0.0.1:8080', '127.0.0.1:80') - - await tunnel.close() - await server.close() - await hs.destroy() -}) - test('basic copy', async function (t) { t.plan(6) const hs = await createHypershell(t) - const server = await createServer(hs) + + const server = hs.createServer({ firewall: null }) + await server.listen() + const transfer = hs.copy(server.publicKey) const dir = await tmp(t) @@ -116,16 +110,176 @@ test('basic copy', async function (t) { await hs.destroy() }) +test('basic tunnel - local forwarding', async function (t) { + t.plan(1) + + const t2 = t.test('tunnel') + + t2.plan(1) + + const hs = await createHypershell(t) + + const server = hs.createServer({ firewall: null }) + await server.listen() + + const tunnel = hs.tunnel(server.publicKey) + + const localPort = await freePort() + const remotePort = await createTcpServer(t, socket => { + socket.on('data', function (data) { + socket.write('Hello World!') + }) + }) + + const proxy1 = await tunnel.local(localPort + ':127.0.0.1', remotePort + ':127.0.0.1') + const socket = net.connect(localPort, '127.0.0.1') + + socket.on('data', function (data) { + t2.alike(data, Buffer.from('Hello World!')) + }) + + socket.write('echo') + + await t2 + + socket.end() + + await new Promise(resolve => socket.on('close', resolve)) + + await proxy1.close() + await tunnel.close() + await server.close() + await hs.destroy() +}) + +test('tunnel allowance', async function (t) { + t.plan(1) + + const t2 = t.test('tunnel') + + t2.plan(1) + + const localPort = await freePort() + const remotePort = await createTcpServer(t, socket => { + t2.pass() + }) + const blockedRemotePort = await createTcpServer(t, socket => { + t2.fail() + }) + + const hs = await createHypershell(t) + + const server = hs.createServer({ + firewall: null, + tunnel: { + allow: ['127.0.0.1:' + remotePort] + } + }) + + // server.tunnel.allow(['127.0.0.1:' + remotePort]) + + await server.listen() + + const tunnel = hs.tunnel(server.publicKey) + + const proxy = await tunnel.local(localPort + ':127.0.0.1', blockedRemotePort + ':127.0.0.1') + + // Fails to connect + const socket = net.connect(localPort, '127.0.0.1') + await new Promise(resolve => socket.on('close', resolve)) + + // Change to the allowed remote port + proxy.forwardTo(remotePort + ':127.0.0.1') + + // Able to connect + const socket2 = net.connect(localPort, '127.0.0.1') + await new Promise(resolve => socket2.on('connect', resolve)) + + await t2 + + socket2.end() + await new Promise(resolve => socket2.on('close', resolve)) + + await proxy.close() + await tunnel.close() + await server.close() + await hs.destroy() +}) + +test.solo('basic tunnel - remote forwarding', async function (t) { + t.plan(1) + + const t2 = t.test('tunnel') + + t2.plan(1) + + const hs = await createHypershell(t) + + const server = hs.createServer({ + firewall: null, + tunnel: { + // allow: ['127.0.0.1:' + remotePort] + } + }) + + await server.listen() + + const tunnel = hs.tunnel(server.publicKey) + + const localPort = await createTcpServer(t, socket => { + socket.on('data', function (data) { + socket.write('Hello World!') + }) + }) + + const remotePort = await freePort() + + const proxy = await tunnel.remote(remotePort + ':127.0.0.1', localPort + ':127.0.0.1') + + // await new Promise(resolve => setTimeout(resolve, 500)) + + const socket = net.connect(remotePort, '127.0.0.1') + + socket.on('data', function (data) { + t2.alike(data, Buffer.from('Hello World!')) + }) + + socket.write('echo') + await t2 + + socket.end() + await new Promise(resolve => socket.on('close', resolve)) + + await proxy.close() + await tunnel.close() + await server.close() + await hs.destroy() +}) + async function createHypershell (t) { const { bootstrap } = await createTestnet(3, t.teardown) return new Hypershell({ bootstrap }) } -async function createServer (hs, opts = {}) { - const server = hs.createServer({ firewall: null, ...opts }) +async function createTcpServer (t, onrequest) { + const server = net.createServer(onrequest) - await server.listen() + t.teardown(() => new Promise(resolve => server.close(resolve))) - return server + await listen(server, 0, '127.0.0.1') + + return server.address().port +} + +function freePort () { + return new Promise(resolve => { + const server = net.createServer() + + server.listen(0, '127.0.0.1', function () { + const addr = server.address() + + server.close(() => resolve(addr.port)) + }) + }) } From 41d2307a38026290098ef8f896430dac1be942b2 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Wed, 16 Oct 2024 21:17:41 -0300 Subject: [PATCH 04/21] Fixes --- bin.js | 2 +- index.js | 4 +- lib/protocols/tunnel.js | 164 +++++++++++++++++++++++++++++++--------- test/lib.js | 36 +++++++-- 4 files changed, 160 insertions(+), 46 deletions(-) diff --git a/bin.js b/bin.js index 4ffc9ac..c5c6551 100755 --- a/bin.js +++ b/bin.js @@ -11,7 +11,7 @@ const main = program .addCommand(require('./bin/server.js')) .addCommand(require('./bin/login.js')) .addCommand(require('./bin/copy.js')) - .addCommand(require('./bin/tunnel.js')) + // .addCommand(require('./bin/tunnel.js')) main.parseAsync().catch(err => { safetyCatch(err) diff --git a/index.js b/index.js index e10eb6b..64a326d 100644 --- a/index.js +++ b/index.js @@ -33,12 +33,10 @@ module.exports = class Hypershell { if (this.protocols.includes('tunnel')) { mux.pair({ protocol: 'hypershell-tunnel' }, () => { - const tunnel = new Tunnel(this.dht, null, { + Tunnel.attach(this.dht, socket.publicKey, { mux, allow: opts.tunnel?.allow }) - - tunnel._createChannel() }) } } diff --git a/lib/protocols/tunnel.js b/lib/protocols/tunnel.js index 763e34b..e085beb 100644 --- a/lib/protocols/tunnel.js +++ b/lib/protocols/tunnel.js @@ -1,4 +1,5 @@ const net = require('net') +const EventEmitter = require('events') const c = require('compact-encoding') const pump = require('pump') const DHT = require('hyperdht') @@ -27,6 +28,8 @@ module.exports = class Tunnel { this.proxies = new Set() this.nextId = nextId() + + this._events = new EventEmitter() } _connect () { @@ -76,28 +79,34 @@ module.exports = class Tunnel { this.channel.open({}) } - onopen (h) {} - - async onclose () { - // TODO - - for (const [, stream] of this.streams) { - stream.destroy() - } - - for (const proxy of this.proxies) { - await proxy.close() - } + static attach (dht, publicKey, opts = {}) { + const tunnel = new this(dht, publicKey, opts) - this.streams.clear() - this.proxies.clear() + tunnel._createChannel() } async local (localAddress, remoteAddress) { let fwd = parseForwardFormat(localAddress + (remoteAddress ? ':' + remoteAddress : '')) + // TODO: This is to get a new isolated channel easly for now + const instance = new Tunnel(this.dht, this.publicKey, { + keyPair: this.keyPair + }) + + // Early connect for faster initial connections, and the opened check + instance._connect() + instance._createChannel() + + const opened = await instance.channel.fullyOpened() + + if (!opened) { + await instance.close() + + throw new Error('Could not connect to server') + } + const server = net.createServer(localSocket => { - this._onLocalConnection(fwd, localSocket) + instance._onLocalConnection(fwd, localSocket) }) await listen(server, fwd.local.port, fwd.local.host) @@ -106,9 +115,11 @@ module.exports = class Tunnel { forwardTo: function (remoteAddress) { fwd = parseForwardFormat(localAddress + ':' + remoteAddress) }, - close: function () { + close: async function () { // TODO: Force close connections - return new Promise(resolve => server.close(resolve)) + await new Promise(resolve => server.close(resolve)) + + await instance.close() } } } @@ -118,7 +129,7 @@ module.exports = class Tunnel { // TODO: This is to get a new isolated channel easly for now // So that if connection is lost then the other side can drop its resources - // Client should handle reconnection e.g. re-execute remote() + // Client can handle reconnection e.g. re-execute remote() const instance = new Tunnel(this.dht, this.publicKey, { keyPair: this.keyPair, allow: [fwd.local.host + ':' + fwd.local.port] @@ -127,27 +138,72 @@ module.exports = class Tunnel { instance._connect() instance._createChannel() + const serverId = instance.nextId() + // Only one remote server per channel instance.wireServer.send({ + id: serverId, port: fwd.remote.port, host: fwd.remote.host, connect: fwd.local }) - await instance.channel.fullyOpened() + const ready = instance._wait(serverId) + const opened = await instance.channel.fullyOpened() + + if (!opened) { + await instance.close() - console.log('fullyOpened') + throw new Error('Could not connect to server') + } + + await ready return { close: async function () { - instance.mux.destroy() - - await instance.channel.fullyClosed() + await instance.close() } } } - async close () {} + async close () { + if (this.mux) { + this.mux.destroy() + + await this.channel.fullyClosed() + } + } + + _wait (id) { + if (!this.channel || this.channel.closed) { + return Promise.reject(new Error('Channel already closed')) + } + + const p = waitForMessage(this._events, id) + + p.catch(safetyCatch) + + return p + } + + onopen (h) {} + + async onclose () { + this._events.emit('close') + + // TODO + + for (const [, stream] of this.streams) { + stream.destroy() + } + + for (const proxy of this.proxies) { + await proxy.close() + } + + this.streams.clear() + this.proxies.clear() + } _onLocalConnection (fwd, localSocket) { this._connect() @@ -167,28 +223,23 @@ module.exports = class Tunnel { }) } - async onMessage (data, c) { - if (data.state === 'WIRE_SERVER_READY') { - this.resolve - } + onMessage (data, c) { + this._events.emit('message', data) } async onWireServer (data, c) { - const { port, host, connect } = data - - console.log('onWireServer A') + const { id, port, host, connect } = data const proxy = await this.local(port + ':' + host, connect.port + ':' + connect.host) - // TODO: Check if channel got closed + if (c.closed) { + await proxy.close().catch(safetyCatch) + return + } this.proxies.add(proxy) - await new Promise(resolve => setTimeout(resolve, 500)) - - this.wireMessage.send({ state: 'WIRE_SERVER_READY' }) - - console.log('onWireServer B') + this.wireMessage.send({ id, message: 'WIRE_SERVER_READY' }) } onWireConnect (data, c) { @@ -310,3 +361,42 @@ function firewallTunnel (addresses, target) { return true } + +function waitForMessage (events, id) { + let waitResolve = null + let waitReject = null + + const promise = new Promise((resolve, reject) => { + waitResolve = resolve + waitReject = reject + }) + + const timeout = setTimeout(() => { + unlisten() + waitReject(new Error('Timed out while waiting for confirmation')) + }, 15000) + + events.on('message', onmessage) + events.on('close', onclose) + + return promise + + function onmessage (data) { + if (data.id === id) { + clearTimeout(timeout) + unlisten() + waitResolve(data) + } + } + + function onclose () { + clearTimeout(timeout) + unlisten() + waitReject(new Error('Connection closed while waiting for confirmation')) + } + + function unlisten () { + events.off('message', onmessage) + events.off('close', onclose) + } +} diff --git a/test/lib.js b/test/lib.js index 720110f..cf73f2c 100644 --- a/test/lib.js +++ b/test/lib.js @@ -176,8 +176,6 @@ test('tunnel allowance', async function (t) { } }) - // server.tunnel.allow(['127.0.0.1:' + remotePort]) - await server.listen() const tunnel = hs.tunnel(server.publicKey) @@ -206,7 +204,7 @@ test('tunnel allowance', async function (t) { await hs.destroy() }) -test.solo('basic tunnel - remote forwarding', async function (t) { +test('basic tunnel - remote forwarding', async function (t) { t.plan(1) const t2 = t.test('tunnel') @@ -236,8 +234,6 @@ test.solo('basic tunnel - remote forwarding', async function (t) { const proxy = await tunnel.remote(remotePort + ':127.0.0.1', localPort + ':127.0.0.1') - // await new Promise(resolve => setTimeout(resolve, 500)) - const socket = net.connect(remotePort, '127.0.0.1') socket.on('data', function (data) { @@ -256,6 +252,36 @@ test.solo('basic tunnel - remote forwarding', async function (t) { await hs.destroy() }) +test('tunnels - failed to connect to server', async function (t) { + t.plan(2) + + const hs = await createHypershell(t) + + const server = hs.createServer({ firewall: null }) + await server.listen() + await server.close() // Server is closed! + + const tunnel = hs.tunnel(server.publicKey) + + try { + await tunnel.local(await freePort(), await freePort()) + t.fail() + } catch (err) { + t.is(err.message, 'Could not connect to server') + } + + try { + await tunnel.remote(await freePort(), await freePort()) + t.fail() + } catch (err) { + t.is(err.message, 'Could not connect to server') + } + + await tunnel.close() + await server.close() + await hs.destroy() +}) + async function createHypershell (t) { const { bootstrap } = await createTestnet(3, t.teardown) From 77129e0e51daecf757c5b79b92f90775a34c551a Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Wed, 16 Oct 2024 21:54:08 -0300 Subject: [PATCH 05/21] Remove unnecessary code --- index.js | 120 +++++++++++++++------------------------- lib/protocols/tunnel.js | 7 +-- 2 files changed, 48 insertions(+), 79 deletions(-) diff --git a/index.js b/index.js index 64a326d..6d8f64c 100644 --- a/index.js +++ b/index.js @@ -2,6 +2,7 @@ const DHT = require('hyperdht') const Protomux = require('protomux') const HypercoreId = require('hypercore-id-encoding') const crypto = require('hypercore-crypto') +const safetyCatch = require('safety-catch') const { ShellServer, ShellClient } = require('./lib/protocols/shell.js') const Copy = require('./lib/protocols/copy.js') const Tunnel = require('./lib/protocols/tunnel.js') @@ -44,18 +45,53 @@ module.exports = class Hypershell { } login (publicKey, opts = {}) { - const client = new Client(this.dht, publicKey, opts) + const socket = this.dht.connect(publicKey, { + keyPair: opts.keyPair || crypto.keyPair(opts.seed), + reusableSocket: true + }) + + socket.setKeepAlive(5000) + + socket.on('error', onSocketError) - return new ShellClient(client.socket, { + return new ShellClient(socket, { rawArgs: opts.rawArgs, stdin: opts.stdin, stdout: opts.stdout }) + + function onSocketError (err) { + if (opts.onerror) { + opts.onerror(err) + } + + if (opts.inherit) { + process.exitCode = 1 + } + + if (!opts.verbose) { + return + } + + if (err.code === 'ECONNRESET') console.error('Connection closed.') + else if (err.code === 'ETIMEDOUT') console.error('Connection timed out.') + else if (err.code === 'PEER_NOT_FOUND') console.error(err.message) + else if (err.code === 'PEER_CONNECTION_FAILED') console.error(err.message, '(probably firewalled)') + else console.error(err) + } } copy (publicKey, opts = {}) { - const client = new Client(this.dht, publicKey, opts) - const mux = Protomux.from(client.socket) + const socket = this.dht.connect(publicKey, { + keyPair: opts.keyPair || crypto.keyPair(opts.seed), + reusableSocket: true + }) + + socket.setKeepAlive(5000) + + socket.on('error', safetyCatch) + + const mux = Protomux.from(socket) return { upload, download, close } @@ -118,20 +154,13 @@ class Server { _onConnection (socket) { this._connections.add(socket) - socket.setKeepAlive(5000) - - socket.on('end', function () { - socket.end() - }) + if (this.verbose) { + console.log('Connection opened', HypercoreId.encode(socket.remotePublicKey)) + } - socket.on('error', function (err) { - if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') { - return - } + socket.setKeepAlive(5000) - // TODO - console.error(err.code, err) - }) + socket.on('error', safetyCatch) socket.on('close', () => { this._connections.delete(socket) @@ -141,10 +170,6 @@ class Server { } }) - if (this.verbose) { - console.log('Connection opened', HypercoreId.encode(socket.remotePublicKey)) - } - if (this._onsocket) { this._onsocket(socket) } @@ -169,61 +194,6 @@ class Server { } } -class Client { - constructor (dht, publicKey, opts = {}) { - this.dht = dht - this.keyPair = opts.keyPair || crypto.keyPair(opts.seed) - this.verbose = !!opts.verbose - this.inherit = !!opts.inherit - - this.socket = this.dht.connect(publicKey, { - keyPair: this.keyPair, - reusableSocket: opts.reusableSocket - }) - - this._onerror = opts.onerror || null - - this._open() - } - - _open () { - this.socket.setKeepAlive(5000) - - this.socket.on('error', this._onSocketError.bind(this)) - this.socket.on('end', this._onSocketEnd.bind(this)) - this.socket.on('close', this._onSocketClose.bind(this)) - } - - _onSocketEnd () { - this.socket.end() - } - - _onSocketError (err) { - if (this._onerror) { - this._onerror(err) - } - - if (this.inherit) { - process.exitCode = 1 - } - - if (!this.verbose) { - return - } - - if (err.code === 'ECONNRESET') console.error('Connection closed.') - else if (err.code === 'ETIMEDOUT') console.error('Connection timed out.') - else if (err.code === 'PEER_NOT_FOUND') console.error(err.message) - else if (err.code === 'PEER_CONNECTION_FAILED') console.error(err.message, '(probably firewalled)') - else console.error(err) - } - - _onSocketClose () { - // TODO: Improve by removing listeners etc - // this.close().catch(safetyCatch) - } -} - function closeConnections (sockets, force) { return new Promise(resolve => { if (sockets.size === 0) { diff --git a/lib/protocols/tunnel.js b/lib/protocols/tunnel.js index e085beb..307f873 100644 --- a/lib/protocols/tunnel.js +++ b/lib/protocols/tunnel.js @@ -1,5 +1,6 @@ const net = require('net') const EventEmitter = require('events') +const crypto = require('hypercore-crypto') const c = require('compact-encoding') const pump = require('pump') const DHT = require('hyperdht') @@ -14,7 +15,7 @@ module.exports = class Tunnel { this.dht = dht this.publicKey = publicKey - this.keyPair = opts.keyPair || null + this.keyPair = opts.keyPair || crypto.keyPair(opts.seed) this.allow = opts.allow || null this.mux = opts.mux || null @@ -191,14 +192,12 @@ module.exports = class Tunnel { async onclose () { this._events.emit('close') - // TODO - for (const [, stream] of this.streams) { stream.destroy() } for (const proxy of this.proxies) { - await proxy.close() + await proxy.close().catch(safetyCatch) } this.streams.clear() From 8d1971434719a934d039289ca51f8dd7824507ba Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Wed, 16 Oct 2024 23:04:48 -0300 Subject: [PATCH 06/21] Reducing indirection --- index.js | 90 +++++++++-------------------- lib/bin/login.js | 12 ++-- lib/protocols/copy.js | 130 ++++++++++++++++++++++++++++++------------ 3 files changed, 127 insertions(+), 105 deletions(-) diff --git a/index.js b/index.js index 6d8f64c..ad71e44 100644 --- a/index.js +++ b/index.js @@ -15,33 +15,32 @@ module.exports = class Hypershell { } createServer (opts = {}) { - return new Server(this.dht, { - ...opts, - onsocket: function (socket) { - const mux = Protomux.from(socket) - - if (this.protocols.includes('shell')) { - mux.pair({ protocol: 'hypershell' }, () => { - ShellServer.attach(mux) - }) - } + return new Server(this.dht, { ...opts, onsocket }) - if (this.protocols.includes('copy')) { - mux.pair({ protocol: 'hypershell-copy' }, () => { - Copy.attach(mux, { permissions: ['pack', 'extract'] }) - }) - } - - if (this.protocols.includes('tunnel')) { - mux.pair({ protocol: 'hypershell-tunnel' }, () => { - Tunnel.attach(this.dht, socket.publicKey, { - mux, - allow: opts.tunnel?.allow - }) + function onsocket (socket) { + const mux = Protomux.from(socket) + + if (this.protocols.includes('shell')) { + mux.pair({ protocol: 'hypershell' }, () => { + ShellServer.attach(mux) + }) + } + + if (this.protocols.includes('copy')) { + mux.pair({ protocol: 'hypershell-copy' }, () => { + Copy.attach({ mux, permissions: ['pack', 'extract'] }) + }) + } + + if (this.protocols.includes('tunnel')) { + mux.pair({ protocol: 'hypershell-tunnel' }, () => { + Tunnel.attach(this.dht, socket.publicKey, { + mux, + allow: opts.tunnel?.allow }) - } + }) } - }) + } } login (publicKey, opts = {}) { @@ -52,7 +51,7 @@ module.exports = class Hypershell { socket.setKeepAlive(5000) - socket.on('error', onSocketError) + socket.on('error', onerror) return new ShellClient(socket, { rawArgs: opts.rawArgs, @@ -60,52 +59,15 @@ module.exports = class Hypershell { stdout: opts.stdout }) - function onSocketError (err) { + function onerror (err) { if (opts.onerror) { opts.onerror(err) } - - if (opts.inherit) { - process.exitCode = 1 - } - - if (!opts.verbose) { - return - } - - if (err.code === 'ECONNRESET') console.error('Connection closed.') - else if (err.code === 'ETIMEDOUT') console.error('Connection timed out.') - else if (err.code === 'PEER_NOT_FOUND') console.error(err.message) - else if (err.code === 'PEER_CONNECTION_FAILED') console.error(err.message, '(probably firewalled)') - else console.error(err) } } copy (publicKey, opts = {}) { - const socket = this.dht.connect(publicKey, { - keyPair: opts.keyPair || crypto.keyPair(opts.seed), - reusableSocket: true - }) - - socket.setKeepAlive(5000) - - socket.on('error', safetyCatch) - - const mux = Protomux.from(socket) - - return { upload, download, close } - - async function upload (source, destination) { - await Copy.upload(mux, source, destination) - } - - async function download (source, destination) { - await Copy.download(mux, source, destination) - } - - async function close () { - mux.destroy() - } + return new Copy(this.dht, publicKey, opts) } tunnel (publicKey, opts = {}) { diff --git a/lib/bin/login.js b/lib/bin/login.js index 06ee794..49c30ab 100644 --- a/lib/bin/login.js +++ b/lib/bin/login.js @@ -24,11 +24,15 @@ module.exports = async function login (keyOrName, opts = {}) { rawArgs: this.rawArgs, stdin: process.stdin, stdout: process.stdout, - verbose: true, - inherit: true - /* onerror: function (err) { + onerror: function (err) { process.exitCode = 1 - } */ + + if (err.code === 'ECONNRESET') console.error('Connection closed.') + else if (err.code === 'ETIMEDOUT') console.error('Connection timed out.') + else if (err.code === 'PEER_NOT_FOUND') console.error(err.message) + else if (err.code === 'PEER_CONNECTION_FAILED') console.error(err.message, '(probably firewalled)') + else console.error(err) + } }) await shell.channel.fullyClosed() diff --git a/lib/protocols/copy.js b/lib/protocols/copy.js index 82e3730..c3cca87 100644 --- a/lib/protocols/copy.js +++ b/lib/protocols/copy.js @@ -1,18 +1,74 @@ const fs = require('fs') const os = require('os') const path = require('path') +const crypto = require('hypercore-crypto') const tar = require('tar-fs') +const Protomux = require('protomux') const c = require('compact-encoding') const m = require('./messages.js') const EMPTY = Buffer.alloc(0) module.exports = class Copy { - constructor (mux, opts = {}) { - this.channel = mux.createChannel({ + constructor (dht, publicKey, opts = {}) { + this.dht = dht || null + this.publicKey = publicKey || null + + this.keyPair = opts.keyPair || crypto.keyPair(opts.seed) + + this.mux = opts.mux || null + this.channel = null + + this.wireHeader = null + this.wireData = null + this.wireError = null + + this.permissions = opts.permissions || [] + this.tar = null + this.error = null + + this._onerror = opts.onerror || null + this._dst = null + + if (!this.mux) { + this._connect() + this._createChannel() + + this.channel.open() + } + } + + static attach (opts = {}) { + const copy = new this(null, null, opts) + + copy._createChannel() + + copy.channel.open() + } + + _connect () { + if (this.mux && !this.mux.stream.destroying) { + return + } + + const socket = this.dht.connect(this.publicKey, { + keyPair: this.keyPair, + reusableSocket: true + }) + + socket.setKeepAlive(5000) + + this.mux = Protomux.from(socket) + } + + _createChannel () { + if (this.channel && this.mux.opened({ protocol: 'hypershell-copy' })) { + return + } + + this.channel = this.mux.createChannel({ protocol: 'hypershell-copy', unique: false, - // handshake: m.handshakeCopy, onopen: this.onopen.bind(this), onclose: this.onclose.bind(this), messages: [ @@ -25,57 +81,57 @@ module.exports = class Copy { this.wireHeader = this.channel.messages[0] this.wireData = this.channel.messages[1] this.wireError = this.channel.messages[2] - - this.permissions = opts.permissions || [] - this.tar = null - this.error = null - - this._onerror = opts.onerror || null - this._dst = null } - static attach (mux, opts) { - const copy = new this(mux, opts) + async upload (source, destination) { + const copy = new Copy(this.dht, this.publicKey, { + keyPair: this.keyPair, + permissions: [], + onerror: this._onerror + }) - copy.channel.open() - } + copy._pack(source, destination) - static async upload (mux, source, destination) { - const copy = new this(mux) + await copy._done() + } - copy.upload(source, destination) + async download (source, destination) { + const copy = new Copy(this.dht, this.publicKey, { + keyPair: this.keyPair, + permissions: ['extract'], + onerror: this._onerror + }) - await copy.channel.fullyClosed() + // The server side should not know either control client's destination + copy._dst = destination + copy.wireHeader.send({ pack: source, destination: null }) - if (copy.error) { - throw makeError(copy.error) - } + await copy._done() } - static async download (mux, source, destination) { - const copy = new this(mux, { permissions: ['extract'] }) - - copy.download(source, destination) + async _done () { + const opened = await this.channel.fullyOpened() - await copy.channel.fullyClosed() + if (!opened) { + await this.close() - if (copy.error) { - throw makeError(copy.error) + if (this.error) throw makeError(this.error) + else throw new Error('Could not connect to server') } - } - upload (source, destination) { - this.channel.open() + await this.channel.fullyClosed() - this._pack(source, destination) + if (this.error) { + throw makeError(this.error) + } } - download (source, destination) { - this.channel.open() + async close () { + if (this.mux) { + this.mux.destroy() - // The server side should not know either control client's destination - this._dst = destination - this.wireHeader.send({ pack: source, destination: null }) + await this.channel.fullyClosed() + } } onopen (h) {} From dae663636c5f45d5ed479fecbe042a42aeb19deb Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Thu, 17 Oct 2024 00:00:58 -0300 Subject: [PATCH 07/21] Adjust things, and add messages --- lib/bin/login.js | 4 ++ lib/protocols/copy.js | 11 ++-- lib/protocols/messages.js | 119 +++++++++++++++++++++++++++++--------- lib/protocols/shell.js | 78 ++++++++++--------------- lib/protocols/tunnel.js | 28 ++++----- test/lib.js | 29 +++++++++- 6 files changed, 167 insertions(+), 102 deletions(-) diff --git a/lib/bin/login.js b/lib/bin/login.js index 49c30ab..bd7d5de 100644 --- a/lib/bin/login.js +++ b/lib/bin/login.js @@ -37,5 +37,9 @@ module.exports = async function login (keyOrName, opts = {}) { await shell.channel.fullyClosed() + if (shell.exitCode !== null) { + process.exitCode = shell.exitCode + } + await hs.destroy() } diff --git a/lib/protocols/copy.js b/lib/protocols/copy.js index c3cca87..a6b45a2 100644 --- a/lib/protocols/copy.js +++ b/lib/protocols/copy.js @@ -69,12 +69,11 @@ module.exports = class Copy { this.channel = this.mux.createChannel({ protocol: 'hypershell-copy', unique: false, - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this), + onclose: this.onWireClose.bind(this), messages: [ - { encoding: m.copyHeader, onmessage: this.onWireHeader.bind(this) }, + { encoding: m.copy.header, onmessage: this.onWireHeader.bind(this) }, { encoding: c.raw, onmessage: this.onWireData.bind(this) }, - { encoding: m.error, onmessage: this.onWireError.bind(this) } + { encoding: m.copy.error, onmessage: this.onWireError.bind(this) } ] }) @@ -134,9 +133,7 @@ module.exports = class Copy { } } - onopen (h) {} - - onclose () { + onWireClose () { if (this.tar) this.tar.destroy() } diff --git a/lib/protocols/messages.js b/lib/protocols/messages.js index c389387..6d4bf2a 100644 --- a/lib/protocols/messages.js +++ b/lib/protocols/messages.js @@ -2,7 +2,9 @@ const c = require('compact-encoding') const stringArray = c.array(c.string) -const handshakeSpawn = { +const shell = exports.shell = {} + +shell.spawn = { preencode (state, s) { c.string.preencode(state, s.command || '') stringArray.preencode(state, s.args || []) @@ -25,24 +27,26 @@ const handshakeSpawn = { } } -const handshakeUpload = { - preencode (state, u) { - c.string.preencode(state, u.target || '') - c.bool.preencode(state, u.isDirectory || false) +shell.resize = { + preencode (state, r) { + c.uint.preencode(state, r.width) + c.uint.preencode(state, r.height) }, - encode (state, u) { - c.string.encode(state, u.target || '') - c.bool.encode(state, u.isDirectory || false) + encode (state, r) { + c.uint.encode(state, r.width) + c.uint.encode(state, r.height) }, decode (state) { return { - target: c.string.decode(state), - isDirectory: c.bool.decode(state) + width: c.uint.decode(state), + height: c.uint.decode(state) } } } -const copyHeader = { +const copy = exports.copy = {} + +copy.header = { preencode (state, h) { c.string.preencode(state, h.pack || '') c.string.preencode(state, h.extract || '') @@ -65,7 +69,7 @@ const copyHeader = { } } -const error = { +copy.error = { preencode (state, e) { c.string.preencode(state, e.code || '') c.string.preencode(state, e.path || '') @@ -85,27 +89,88 @@ const error = { } } -const resize = { - preencode (state, r) { - c.uint.preencode(state, r.width) - c.uint.preencode(state, r.height) +const tunnel = exports.tunnel = {} + +tunnel.message = { + preencode (state, h) { + c.uint.preencode(state, h.id) + c.string.preencode(state, h.message) }, - encode (state, r) { - c.uint.encode(state, r.width) - c.uint.encode(state, r.height) + encode (state, h) { + c.uint.encode(state, h.id) + c.string.encode(state, h.message) }, decode (state) { return { - width: c.uint.decode(state), - height: c.uint.decode(state) + id: c.uint.decode(state), + message: c.string.decode(state) + } + } +} + +tunnel.server = { + preencode (state, h) { + c.uint.preencode(state, h.id) + c.uint.preencode(state, h.port) + c.string.preencode(state, h.host || '') + c.uint.preencode(state, h.connect.port) + c.string.preencode(state, h.connect.host || '') + }, + encode (state, h) { + c.uint.encode(state, h.id) + c.uint.encode(state, h.port) + c.string.encode(state, h.host || '') + c.uint.encode(state, h.connect.port) + c.string.encode(state, h.connect.host || '') + }, + decode (state) { + return { + id: c.uint.decode(state), + port: c.uint.decode(state), + host: c.string.decode(state), + connect: { + port: c.uint.decode(state), + host: c.string.decode(state) + } } } } -module.exports = { - handshakeSpawn, - handshakeUpload, - copyHeader, - error, - resize +tunnel.connect = { + preencode (state, h) { + c.uint.preencode(state, h.clientId) + c.uint.preencode(state, h.connect.port) + c.string.preencode(state, h.connect.host) + }, + encode (state, h) { + c.uint.encode(state, h.clientId) + c.uint.encode(state, h.connect.port) + c.string.encode(state, h.connect.host) + }, + decode (state) { + return { + clientId: c.uint.decode(state), + connect: { + port: c.uint.decode(state), + host: c.string.decode(state) + } + } + } +} + +tunnel.pump = { + preencode (state, h) { + c.uint.preencode(state, h.clientId) + c.uint.preencode(state, h.serverId) + }, + encode (state, h) { + c.uint.encode(state, h.clientId) + c.uint.encode(state, h.serverId) + }, + decode (state) { + return { + clientId: c.uint.decode(state), + serverId: c.uint.decode(state) + } + } } diff --git a/lib/protocols/shell.js b/lib/protocols/shell.js index 9c1ba45..03957fe 100644 --- a/lib/protocols/shell.js +++ b/lib/protocols/shell.js @@ -4,7 +4,6 @@ const c = require('compact-encoding') const PTY = require('tt-native') const { PassThrough } = require('streamx') const m = require('./messages.js') -const waitForSocket = require('../wait-for-socket.js') const isWin = os.platform() === 'win32' const shellFile = isWin ? 'powershell.exe' : (process.env.SHELL || 'bash') @@ -14,22 +13,21 @@ class ShellServer { constructor (mux) { this.channel = mux.createChannel({ protocol: 'hypershell', - id: null, - handshake: m.handshakeSpawn, - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this), + handshake: m.shell.spawn, + onopen: this.onWireOpen.bind(this), + onclose: this.onWireClose.bind(this), messages: [ - { encoding: c.buffer, onmessage: this.onstdin.bind(this) }, + { encoding: c.buffer, onmessage: this.onWireStdin.bind(this) }, { encoding: c.buffer }, // stdout { encoding: c.buffer }, // stderr { encoding: c.uint }, // exit code - { encoding: m.resize, onmessage: this.onresize.bind(this) } + { encoding: m.shell.resize, onmessage: this.onWireResize.bind(this) } ] }) - if (!this.channel) { - throw new Error('Channel duplicated') - } + this.wireStdout = this.channel.messages[1] + this.wireStderr = this.channel.messages[2] + this.wireExitCode = this.channel.messages[3] this.pty = null @@ -40,7 +38,7 @@ class ShellServer { return new this(mux) } - onopen (handshake) { + onWireOpen (handshake) { try { this.pty = PTY.spawn(handshake.command || shellFile, handshake.args, { cwd: os.homedir(), @@ -49,18 +47,18 @@ class ShellServer { height: handshake.height }) } catch (err) { - this.channel.messages[3].send(1) - this.channel.messages[2].send(Buffer.from(err.toString() + '\n')) + this.wireExitCode.send(1) + this.wireStderr.send(Buffer.from(err.toString() + '\n')) this.channel.close() return } - this.pty.on('data', (data) => this.channel.messages[1].send(data)) - this.pty.once('exit', (code) => this.channel.messages[3].send(code)) + this.pty.on('data', (data) => this.wireStdout.send(data)) + this.pty.once('exit', (code) => this.wireExitCode.send(code)) this.pty.once('close', () => this.channel.close()) } - onclose () { + onWireClose () { if (this.pty) { try { this.pty.kill('SIGKILL') @@ -68,12 +66,12 @@ class ShellServer { } } - onstdin (data, c) { + onWireStdin (data, c) { if (data === null) this.pty.write(EMPTY) else this.pty.write(data) } - onresize (data, c) { + onWireResize (data, c) { this.pty.resize(data.width, data.height) } } @@ -91,21 +89,19 @@ class ShellClient { this.channel = this.mux.createChannel({ protocol: 'hypershell', id: null, - handshake: m.handshakeSpawn, - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this), + handshake: m.shell.spawn, + onclose: this.onWireClose.bind(this), messages: [ { encoding: c.buffer }, // stdin - { encoding: c.buffer, onmessage: this.onstdout.bind(this) }, - { encoding: c.buffer, onmessage: this.onstderr.bind(this) }, - { encoding: c.uint, onmessage: this.onexitcode.bind(this) }, - { encoding: m.resize } + { encoding: c.buffer, onmessage: this.onWireStdout.bind(this) }, + { encoding: c.buffer, onmessage: this.onWireStderr.bind(this) }, + { encoding: c.uint, onmessage: this.onWireExitCode.bind(this) }, + { encoding: m.shell.resize } ] }) - if (!this.channel) { - throw new Error('Channel duplicated') - } + this.wireStdin = this.channel.messages[0] + this.wireResize = this.channel.messages[4] this.stdin = opts.stdin || new PassThrough() this.stdout = opts.stdout || new PassThrough() @@ -130,21 +126,14 @@ class ShellClient { } async close () { - this.socket.destroy() - - // TODO: Not needed anymore? - await waitForSocket(this.socket) + this.channel.close() await this.channel.fullyClosed() - } - destroy () { this.socket.destroy() } - onopen () {} - - onclose () { + onWireClose () { this.socket.destroy() } @@ -165,28 +154,23 @@ class ShellClient { data = Buffer.from(data) } - this.channel.messages[0].send(data) + this.wireStdin.send(data) } - onstdout (data, c) { + onWireStdout (data, c) { this.stdout.write(data) } - onstderr (data, c) { + onWireStderr (data, c) { this.stderr.write(data) } - onexitcode (code, c) { + onWireExitCode (code, c) { this.exitCode = code - - // TODO - if (this.stdin === process.stdin) { - process.exitCode = code - } } onresize () { - this.channel.messages[4].send({ + this.wireResize.send({ width: this.stdout.columns || 80, height: this.stdout.rows || 24 }) diff --git a/lib/protocols/tunnel.js b/lib/protocols/tunnel.js index 307f873..16868b6 100644 --- a/lib/protocols/tunnel.js +++ b/lib/protocols/tunnel.js @@ -9,6 +9,7 @@ const SecretStream = require('@hyperswarm/secret-stream') const listen = require('listen-async') const safetyCatch = require('safety-catch') const nextId = require('../next-id') +const m = require('./messages.js') module.exports = class Tunnel { constructor (dht, publicKey, opts = {}) { @@ -53,25 +54,18 @@ module.exports = class Tunnel { return } - const channel = this.mux.createChannel({ + this.channel = this.mux.createChannel({ protocol: 'hypershell-tunnel', handshake: c.json, - onopen: this.onopen.bind(this), - onclose: this.onclose.bind(this), + onclose: this.onWireClose.bind(this), messages: [ - { encoding: c.json, onmessage: this.onMessage.bind(this) }, - { encoding: c.json, onmessage: this.onWireServer.bind(this) }, - { encoding: c.json, onmessage: this.onWireConnect.bind(this) }, - { encoding: c.json, onmessage: this.onWirePump.bind(this) } + { encoding: m.tunnel.message, onmessage: this.onWireMessage.bind(this) }, + { encoding: m.tunnel.server, onmessage: this.onWireServer.bind(this) }, + { encoding: m.tunnel.connect, onmessage: this.onWireConnect.bind(this) }, + { encoding: m.tunnel.pump, onmessage: this.onWirePump.bind(this) } ] }) - if (channel === null) { - return - } - - this.channel = channel - this.wireMessage = this.channel.messages[0] this.wireServer = this.channel.messages[1] this.wireConnect = this.channel.messages[2] @@ -94,7 +88,7 @@ module.exports = class Tunnel { keyPair: this.keyPair }) - // Early connect for faster initial connections, and the opened check + // Early connect for faster initial connections also instance._connect() instance._createChannel() @@ -187,9 +181,7 @@ module.exports = class Tunnel { return p } - onopen (h) {} - - async onclose () { + async onWireClose () { this._events.emit('close') for (const [, stream] of this.streams) { @@ -222,7 +214,7 @@ module.exports = class Tunnel { }) } - onMessage (data, c) { + onWireMessage (data, c) { this._events.emit('message', data) } diff --git a/test/lib.js b/test/lib.js index cf73f2c..9500b6b 100644 --- a/test/lib.js +++ b/test/lib.js @@ -49,16 +49,16 @@ test('basic shell', async function (t) { t.absent(getStreamError(shell.socket)) }) -test('server is closed first', async function (t) { +test('shell - server is closed first', async function (t) { const hs = await createHypershell(t) const server = hs.createServer({ firewall: null }) await server.listen() - const shell = hs.login(server.publicKey) + let exitCode = null + const shell = hs.login(server.publicKey, { onerror }) await shell.ready() - await server.close() await shell.channel.fullyClosed() await hs.destroy() @@ -66,6 +66,29 @@ test('server is closed first', async function (t) { const err = getStreamError(shell.socket) t.is(err.code, 'ECONNRESET') + t.is(exitCode, 1) + + function onerror () { + exitCode = 1 + } +}) + +test('shell - exit code', async function (t) { + const hs = await createHypershell(t) + + const server = hs.createServer({ firewall: null }) + await server.listen() + + const shell = hs.login(server.publicKey) + await shell.ready() + + shell.stdin.write('exit 127\n') + + await shell.channel.fullyClosed() + t.is(shell.exitCode, 127) + + await server.close() + await hs.destroy() }) test('basic copy', async function (t) { From 413f8c7e06c084e458a847a0222446f7efc2932a Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Fri, 18 Oct 2024 02:24:54 -0300 Subject: [PATCH 08/21] Fix keygen option --- lib/bin/copy.js | 2 +- lib/bin/login.js | 2 +- lib/bin/server.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bin/copy.js b/lib/bin/copy.js index 1ee6b25..dbf861b 100644 --- a/lib/bin/copy.js +++ b/lib/bin/copy.js @@ -12,7 +12,7 @@ module.exports = async function copy (source, destination, opts = {}) { const keyFilename = path.resolve(opts.f || path.join(constants.dir, 'id')) if (!fs.existsSync(keyFilename)) { - await keygen({ filename: keyFilename }) + await keygen({ f: keyFilename }) } const direction = source[0] === '@' || publicKeyExpr.test(source) ? 'download' : 'upload' diff --git a/lib/bin/login.js b/lib/bin/login.js index bd7d5de..e4f651f 100644 --- a/lib/bin/login.js +++ b/lib/bin/login.js @@ -10,7 +10,7 @@ module.exports = async function login (keyOrName, opts = {}) { const keyFilename = path.resolve(opts.f || path.join(constants.dir, 'id')) if (!fs.existsSync(keyFilename)) { - await keygen({ filename: keyFilename }) + await keygen({ f: keyFilename }) } const serverPublicKey = await getKnownPeer(keyOrName, { verbose: true }) diff --git a/lib/bin/server.js b/lib/bin/server.js index 28cf35d..7773637 100644 --- a/lib/bin/server.js +++ b/lib/bin/server.js @@ -15,7 +15,7 @@ module.exports = async function server (opts = {}) { const firewallEnabled = !opts.disableFirewall if (!fs.existsSync(keyFilename)) { - await keygen({ filename: keyFilename }) + await keygen({ f: keyFilename }) } if (firewallEnabled && !fs.existsSync(firewallFilename)) { From 3bb100c212a3e1debf7e8ee59e241fa7f9fa11e1 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Fri, 18 Oct 2024 07:36:05 -0300 Subject: [PATCH 09/21] Fix reusability of tunnels instance (fixes profound bugs) --- bin.js | 2 +- bin/tunnel.js | 10 ++ lib/bin/tunnel.js | 50 ++++++++++ lib/next-id.js | 11 --- lib/protocols/messages.js | 49 +++++++--- lib/protocols/tunnel.js | 152 +++++++++++++++++------------ package.json | 2 +- test/lib.js | 199 +++++++++++++++++++++++++++++--------- 8 files changed, 337 insertions(+), 138 deletions(-) create mode 100644 bin/tunnel.js create mode 100644 lib/bin/tunnel.js delete mode 100644 lib/next-id.js diff --git a/bin.js b/bin.js index c5c6551..4ffc9ac 100755 --- a/bin.js +++ b/bin.js @@ -11,7 +11,7 @@ const main = program .addCommand(require('./bin/server.js')) .addCommand(require('./bin/login.js')) .addCommand(require('./bin/copy.js')) - // .addCommand(require('./bin/tunnel.js')) + .addCommand(require('./bin/tunnel.js')) main.parseAsync().catch(err => { safetyCatch(err) diff --git a/bin/tunnel.js b/bin/tunnel.js new file mode 100644 index 0000000..9263a56 --- /dev/null +++ b/bin/tunnel.js @@ -0,0 +1,10 @@ +const { createCommand } = require('commander') + +module.exports = createCommand('tunnel') + .description('port forwarding') + .argument('', 'public key or name of the server') + .option('-L <[address:]port:host:hostport...>', 'local port forwarding') + .option('-R <[address:]port:host:hostport...>', 'remote port forwarding') + .option('-f ', 'filename of the seed key file') + .option('--bootstrap ', 'custom dht nodes') + .action(require('../lib/bin/tunnel.js')) diff --git a/lib/bin/tunnel.js b/lib/bin/tunnel.js new file mode 100644 index 0000000..dbfa892 --- /dev/null +++ b/lib/bin/tunnel.js @@ -0,0 +1,50 @@ +const fs = require('fs') +const path = require('path') +const goodbye = require('graceful-goodbye') +const constants = require('../constants.js') +const keygen = require('./keygen.js') +const getKnownPeer = require('../get-known-peer.js') +const fileToKeyPair = require('../file-to-keypair.js') +const Hypershell = require('../../index.js') + +module.exports = async function tunnel (keyOrName, opts = {}) { + const keyFilename = path.resolve(opts.f || path.join(constants.dir, 'id')) + + if (!fs.existsSync(keyFilename)) { + await keygen({ f: keyFilename }) + } + + const serverPublicKey = await getKnownPeer(keyOrName, { verbose: true }) + + const hs = new Hypershell({ bootstrap: opts.bootstrap }) + + const tunnel = hs.tunnel(serverPublicKey, { + keyPair: await fileToKeyPair(keyFilename) + }) + + let proxy = null + + if (opts.L) { + proxy = await tunnel.local(opts.L) + } else if (opts.R) { + proxy = await tunnel.remote(opts.R) + } else { + throw new Error('-L o -R is required') + } + + console.log('Tunnel is ready!') + + const unregister = goodbye(close) + + return async function cleanup () { + unregister() + + await close() + } + + async function close () { + await proxy.close() + await tunnel.close() + await hs.destroy() + } +} diff --git a/lib/next-id.js b/lib/next-id.js deleted file mode 100644 index 7a03074..0000000 --- a/lib/next-id.js +++ /dev/null @@ -1,11 +0,0 @@ -module.exports = function nextId () { - let id = 1 - - return function () { - if (id === 0xffffffff) { - id = 1 - } - - return id++ - } -} diff --git a/lib/protocols/messages.js b/lib/protocols/messages.js index 6d4bf2a..80a29f1 100644 --- a/lib/protocols/messages.js +++ b/lib/protocols/messages.js @@ -93,31 +93,31 @@ const tunnel = exports.tunnel = {} tunnel.message = { preencode (state, h) { - c.uint.preencode(state, h.id) + c.string.preencode(state, h.id) c.string.preencode(state, h.message) }, encode (state, h) { - c.uint.encode(state, h.id) + c.string.encode(state, h.id) c.string.encode(state, h.message) }, decode (state) { return { - id: c.uint.decode(state), + id: c.string.decode(state), message: c.string.decode(state) } } } -tunnel.server = { +tunnel.serverStart = { preencode (state, h) { - c.uint.preencode(state, h.id) + c.string.preencode(state, h.id) c.uint.preencode(state, h.port) c.string.preencode(state, h.host || '') c.uint.preencode(state, h.connect.port) c.string.preencode(state, h.connect.host || '') }, encode (state, h) { - c.uint.encode(state, h.id) + c.string.encode(state, h.id) c.uint.encode(state, h.port) c.string.encode(state, h.host || '') c.uint.encode(state, h.connect.port) @@ -125,7 +125,7 @@ tunnel.server = { }, decode (state) { return { - id: c.uint.decode(state), + id: c.string.decode(state), port: c.uint.decode(state), host: c.string.decode(state), connect: { @@ -136,20 +136,37 @@ tunnel.server = { } } +tunnel.serverClose = { + preencode (state, h) { + c.string.preencode(state, h.id) + }, + encode (state, h) { + c.string.encode(state, h.id) + }, + decode (state) { + return { + id: c.string.decode(state) + } + } +} + tunnel.connect = { preencode (state, h) { - c.uint.preencode(state, h.clientId) + c.string.preencode(state, h.serverId || '') + c.uint.preencode(state, h.localStreamId) c.uint.preencode(state, h.connect.port) c.string.preencode(state, h.connect.host) }, encode (state, h) { - c.uint.encode(state, h.clientId) + c.string.encode(state, h.serverId || '') + c.uint.encode(state, h.localStreamId) c.uint.encode(state, h.connect.port) c.string.encode(state, h.connect.host) }, decode (state) { return { - clientId: c.uint.decode(state), + serverId: c.string.decode(state), + localStreamId: c.uint.decode(state), connect: { port: c.uint.decode(state), host: c.string.decode(state) @@ -160,17 +177,17 @@ tunnel.connect = { tunnel.pump = { preencode (state, h) { - c.uint.preencode(state, h.clientId) - c.uint.preencode(state, h.serverId) + c.uint.preencode(state, h.localStreamId) + c.uint.preencode(state, h.remoteStreamId) }, encode (state, h) { - c.uint.encode(state, h.clientId) - c.uint.encode(state, h.serverId) + c.uint.encode(state, h.localStreamId) + c.uint.encode(state, h.remoteStreamId) }, decode (state) { return { - clientId: c.uint.decode(state), - serverId: c.uint.decode(state) + localStreamId: c.uint.decode(state), + remoteStreamId: c.uint.decode(state) } } } diff --git a/lib/protocols/tunnel.js b/lib/protocols/tunnel.js index 16868b6..e5e42b6 100644 --- a/lib/protocols/tunnel.js +++ b/lib/protocols/tunnel.js @@ -6,9 +6,8 @@ const pump = require('pump') const DHT = require('hyperdht') const Protomux = require('protomux') const SecretStream = require('@hyperswarm/secret-stream') -const listen = require('listen-async') +const bind = require('like-bind') const safetyCatch = require('safety-catch') -const nextId = require('../next-id') const m = require('./messages.js') module.exports = class Tunnel { @@ -17,7 +16,7 @@ module.exports = class Tunnel { this.publicKey = publicKey this.keyPair = opts.keyPair || crypto.keyPair(opts.seed) - this.allow = opts.allow || null + this.allow = opts.allow || [] this.mux = opts.mux || null this.channel = null @@ -26,10 +25,9 @@ module.exports = class Tunnel { this.wireStream = null this.wirePump = null + this._allow = new Map() this.streams = new Map() - this.proxies = new Set() - - this.nextId = nextId() + this.proxies = new Map() this._events = new EventEmitter() } @@ -50,7 +48,7 @@ module.exports = class Tunnel { } _createChannel () { - if (this.mux.opened({ protocol: 'hypershell-tunnel' })) { + if (this.channel && this.mux.opened({ protocol: 'hypershell-tunnel' })) { return } @@ -60,16 +58,18 @@ module.exports = class Tunnel { onclose: this.onWireClose.bind(this), messages: [ { encoding: m.tunnel.message, onmessage: this.onWireMessage.bind(this) }, - { encoding: m.tunnel.server, onmessage: this.onWireServer.bind(this) }, + { encoding: m.tunnel.serverStart, onmessage: this.onWireServerStart.bind(this) }, + { encoding: m.tunnel.serverClose, onmessage: this.onWireServerClose.bind(this) }, { encoding: m.tunnel.connect, onmessage: this.onWireConnect.bind(this) }, { encoding: m.tunnel.pump, onmessage: this.onWirePump.bind(this) } ] }) this.wireMessage = this.channel.messages[0] - this.wireServer = this.channel.messages[1] - this.wireConnect = this.channel.messages[2] - this.wirePump = this.channel.messages[3] + this.wireServerStart = this.channel.messages[1] + this.wireServerClose = this.channel.messages[2] + this.wireConnect = this.channel.messages[3] + this.wirePump = this.channel.messages[4] this.channel.open({}) } @@ -77,44 +77,41 @@ module.exports = class Tunnel { static attach (dht, publicKey, opts = {}) { const tunnel = new this(dht, publicKey, opts) + if (opts.allow) { + tunnel.allow = opts.allow + } else { + // Allow the client to tell the server to connect to anything + tunnel.allow = null + } + tunnel._createChannel() } - async local (localAddress, remoteAddress) { + async local (localAddress, remoteAddress, opts = {}) { let fwd = parseForwardFormat(localAddress + (remoteAddress ? ':' + remoteAddress : '')) - // TODO: This is to get a new isolated channel easly for now - const instance = new Tunnel(this.dht, this.publicKey, { - keyPair: this.keyPair - }) - // Early connect for faster initial connections also - instance._connect() - instance._createChannel() + this._connect() + this._createChannel() - const opened = await instance.channel.fullyOpened() + const opened = await this.channel.fullyOpened() if (!opened) { - await instance.close() - throw new Error('Could not connect to server') } const server = net.createServer(localSocket => { - instance._onLocalConnection(fwd, localSocket) + this._onLocalConnection(fwd, localSocket, opts.serverId) }) - await listen(server, fwd.local.port, fwd.local.host) + await bind.listen(server, fwd.local.port, fwd.local.host) return { forwardTo: function (remoteAddress) { fwd = parseForwardFormat(localAddress + ':' + remoteAddress) }, close: async function () { - // TODO: Force close connections - await new Promise(resolve => server.close(resolve)) - - await instance.close() + await bind.close(server, { force: true }) } } } @@ -122,32 +119,35 @@ module.exports = class Tunnel { async remote (remoteAddress, localAddress) { const fwd = parseForwardFormat((localAddress ? localAddress + ':' : '') + remoteAddress) - // TODO: This is to get a new isolated channel easly for now - // So that if connection is lost then the other side can drop its resources - // Client can handle reconnection e.g. re-execute remote() - const instance = new Tunnel(this.dht, this.publicKey, { - keyPair: this.keyPair, - allow: [fwd.local.host + ':' + fwd.local.port] - }) + // TODO: Maybe pass two args to parseForwardFormat + if (!localAddress) { + const tmp = fwd.local + + fwd.local = fwd.remote + fwd.remote = tmp + } + + this._connect() + this._createChannel() - instance._connect() - instance._createChannel() + // Long secret id so the server can't guess the local allow + const serverId = crypto.randomBytes(32).toString('hex') - const serverId = instance.nextId() + this._allow.set(serverId, [fwd.local.host + ':' + fwd.local.port]) - // Only one remote server per channel - instance.wireServer.send({ + this.wireServerStart.send({ id: serverId, port: fwd.remote.port, host: fwd.remote.host, connect: fwd.local }) - const ready = instance._wait(serverId) - const opened = await instance.channel.fullyOpened() + const ready = this._wait(serverId) + const opened = await this.channel.fullyOpened() if (!opened) { - await instance.close() + // TODO: Check if onWireClose gets triggered if couldn't connect + this._allow.delete(serverId) throw new Error('Could not connect to server') } @@ -155,8 +155,20 @@ module.exports = class Tunnel { await ready return { - close: async function () { - await instance.close() + close: async () => { + this._allow.delete(serverId) + + if (this.mux.stream.destroying || this.channel.closed) { + return + } + + this.wireServerClose.send({ + id: serverId + }) + + const closed = this._wait(serverId) + + await closed } } } @@ -188,7 +200,7 @@ module.exports = class Tunnel { stream.destroy() } - for (const proxy of this.proxies) { + for (const [, proxy] of this.proxies) { await proxy.close().catch(safetyCatch) } @@ -196,7 +208,7 @@ module.exports = class Tunnel { this.proxies.clear() } - _onLocalConnection (fwd, localSocket) { + _onLocalConnection (fwd, localSocket, serverId) { this._connect() this._createChannel() @@ -209,7 +221,8 @@ module.exports = class Tunnel { localSocket.on('error', safetyCatch) this.wireConnect.send({ - clientId: rawStream.id, + serverId, + localStreamId: rawStream.id, connect: fwd.remote }) } @@ -218,31 +231,48 @@ module.exports = class Tunnel { this._events.emit('message', data) } - async onWireServer (data, c) { + async onWireServerStart (data, c) { const { id, port, host, connect } = data - const proxy = await this.local(port + ':' + host, connect.port + ':' + connect.host) + const proxy = await this.local(port + ':' + host, connect.port + ':' + connect.host, { serverId: id }) if (c.closed) { await proxy.close().catch(safetyCatch) return } - this.proxies.add(proxy) + this.proxies.set(id, proxy) + + this.wireMessage.send({ id, message: 'WIRE_SERVER_LISTENING' }) + } + + async onWireServerClose (data, c) { + const { id } = data + + const proxy = this.proxies.get(id) + + if (proxy) { + this.proxies.delete(id) - this.wireMessage.send({ id, message: 'WIRE_SERVER_READY' }) + await proxy.close().catch(safetyCatch) + } + + this.wireMessage.send({ id, message: 'WIRE_SERVER_CLOSED' }) } onWireConnect (data, c) { - const { clientId, connect } = data + const { serverId, localStreamId, connect } = data + + const isLocallyAllowed = this._allow.has(serverId) && !firewallTunnel(this._allow.get(serverId), connect) + const isGloballyAllowed = !firewallTunnel(this.allow, connect) - if (firewallTunnel(this.allow, connect)) { + if (!(isLocallyAllowed || isGloballyAllowed)) { c.close() return } const rawStream = this._createRawStream() - const secretStream = this._connectRawStream(rawStream, clientId, true) + const secretStream = this._connectRawStream(rawStream, localStreamId, true) const remoteSocket = net.connect(connect.port, connect.host) rawStream.userData = { secretStream, remoteSocket } @@ -250,23 +280,23 @@ module.exports = class Tunnel { pump(secretStream, remoteSocket, secretStream) this.wirePump.send({ - clientId, - serverId: rawStream.id + localStreamId, + remoteStreamId: rawStream.id }) } onWirePump (data, c) { - const { clientId, serverId } = data + const { localStreamId, remoteStreamId } = data - const rawStream = this.streams.get(clientId) + const rawStream = this.streams.get(localStreamId) if (!rawStream) { - throw new Error('Stream not found: ' + clientId) + throw new Error('Stream not found: ' + localStreamId) } const { localSocket } = rawStream.userData - const secretStream = this._connectRawStream(rawStream, serverId, false) + const secretStream = this._connectRawStream(rawStream, remoteStreamId, false) rawStream.userData.secretStream = secretStream diff --git a/package.json b/package.json index 130fde8..1888a4c 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "hypercore-crypto": "^3.4.2", "hypercore-id-encoding": "^1.3.0", "hyperdht": "^6.19.0", - "listen-async": "^1.0.0", + "like-bind": "^1.0.0", "protomux": "^3.10.0", "protomux-rpc": "^1.6.0", "pump": "^3.0.2", diff --git a/test/lib.js b/test/lib.js index 9500b6b..2453bcd 100644 --- a/test/lib.js +++ b/test/lib.js @@ -6,7 +6,7 @@ const tmp = require('like-tmp') const crypto = require('hypercore-crypto') const { getStreamError } = require('streamx') const createTestnet = require('hyperdht/testnet') -const listen = require('listen-async') +const bind = require('like-bind') const Hypershell = require('../index.js') test('basic shell', async function (t) { @@ -16,15 +16,13 @@ test('basic shell', async function (t) { t2.plan(1) - const { bootstrap } = await createTestnet(3, t.teardown) - const hs = new Hypershell({ bootstrap }) - - const clientKeyPair = crypto.keyPair() + const [hs1, hs2] = await createHypershells(t) + const keyPair = crypto.keyPair() - const server = hs.createServer({ firewall: [clientKeyPair.publicKey] }) + const server = hs1.createServer({ firewall: [keyPair.publicKey] }) await server.listen() - const shell = hs.login(server.publicKey, { keyPair: clientKeyPair }) + const shell = hs2.login(server.publicKey, { keyPair }) let out = '' @@ -44,24 +42,23 @@ test('basic shell', async function (t) { await shell.close() await server.close() - await hs.destroy() t.absent(getStreamError(shell.socket)) }) test('shell - server is closed first', async function (t) { - const hs = await createHypershell(t) + const [hs1, hs2] = await createHypershells(t) + const keyPair = crypto.keyPair() - const server = hs.createServer({ firewall: null }) + const server = hs1.createServer({ firewall: [keyPair.publicKey] }) await server.listen() let exitCode = null - const shell = hs.login(server.publicKey, { onerror }) + const shell = hs2.login(server.publicKey, { keyPair, onerror }) await shell.ready() await server.close() await shell.channel.fullyClosed() - await hs.destroy() const err = getStreamError(shell.socket) @@ -74,12 +71,13 @@ test('shell - server is closed first', async function (t) { }) test('shell - exit code', async function (t) { - const hs = await createHypershell(t) + const [hs1, hs2] = await createHypershells(t) + const keyPair = crypto.keyPair() - const server = hs.createServer({ firewall: null }) + const server = hs1.createServer({ firewall: [keyPair.publicKey] }) await server.listen() - const shell = hs.login(server.publicKey) + const shell = hs2.login(server.publicKey, { keyPair }) await shell.ready() shell.stdin.write('exit 127\n') @@ -88,18 +86,18 @@ test('shell - exit code', async function (t) { t.is(shell.exitCode, 127) await server.close() - await hs.destroy() }) test('basic copy', async function (t) { t.plan(6) - const hs = await createHypershell(t) + const [hs1, hs2] = await createHypershells(t) + const keyPair = crypto.keyPair() - const server = hs.createServer({ firewall: null }) + const server = hs1.createServer({ firewall: [keyPair.publicKey] }) await server.listen() - const transfer = hs.copy(server.publicKey) + const transfer = hs2.copy(server.publicKey, { keyPair }) const dir = await tmp(t) const msg = Buffer.from('Hello World!') @@ -130,7 +128,6 @@ test('basic copy', async function (t) { await transfer.close() await server.close() - await hs.destroy() }) test('basic tunnel - local forwarding', async function (t) { @@ -140,12 +137,13 @@ test('basic tunnel - local forwarding', async function (t) { t2.plan(1) - const hs = await createHypershell(t) + const [hs1, hs2] = await createHypershells(t) + const keyPair = crypto.keyPair() - const server = hs.createServer({ firewall: null }) + const server = hs1.createServer({ firewall: [keyPair.publicKey] }) await server.listen() - const tunnel = hs.tunnel(server.publicKey) + const tunnel = hs2.tunnel(server.publicKey, { keyPair }) const localPort = await freePort() const remotePort = await createTcpServer(t, socket => { @@ -162,17 +160,14 @@ test('basic tunnel - local forwarding', async function (t) { }) socket.write('echo') - await t2 socket.end() - await new Promise(resolve => socket.on('close', resolve)) await proxy1.close() await tunnel.close() await server.close() - await hs.destroy() }) test('tunnel allowance', async function (t) { @@ -190,10 +185,11 @@ test('tunnel allowance', async function (t) { t2.fail() }) - const hs = await createHypershell(t) + const [hs1, hs2] = await createHypershells(t) + const keyPair = crypto.keyPair() - const server = hs.createServer({ - firewall: null, + const server = hs1.createServer({ + firewall: [keyPair.publicKey], tunnel: { allow: ['127.0.0.1:' + remotePort] } @@ -201,7 +197,7 @@ test('tunnel allowance', async function (t) { await server.listen() - const tunnel = hs.tunnel(server.publicKey) + const tunnel = hs2.tunnel(server.publicKey, { keyPair }) const proxy = await tunnel.local(localPort + ':127.0.0.1', blockedRemotePort + ':127.0.0.1') @@ -224,7 +220,6 @@ test('tunnel allowance', async function (t) { await proxy.close() await tunnel.close() await server.close() - await hs.destroy() }) test('basic tunnel - remote forwarding', async function (t) { @@ -234,18 +229,13 @@ test('basic tunnel - remote forwarding', async function (t) { t2.plan(1) - const hs = await createHypershell(t) - - const server = hs.createServer({ - firewall: null, - tunnel: { - // allow: ['127.0.0.1:' + remotePort] - } - }) + const [hs1, hs2] = await createHypershells(t) + const keyPair = crypto.keyPair() + const server = hs1.createServer({ firewall: [keyPair.publicKey] }) await server.listen() - const tunnel = hs.tunnel(server.publicKey) + const tunnel = hs2.tunnel(server.publicKey, { keyPair }) const localPort = await createTcpServer(t, socket => { socket.on('data', function (data) { @@ -272,19 +262,19 @@ test('basic tunnel - remote forwarding', async function (t) { await proxy.close() await tunnel.close() await server.close() - await hs.destroy() }) test('tunnels - failed to connect to server', async function (t) { t.plan(2) - const hs = await createHypershell(t) + const [hs1, hs2] = await createHypershells(t) + const keyPair = crypto.keyPair() - const server = hs.createServer({ firewall: null }) + const server = hs1.createServer({ firewall: [keyPair.publicKey] }) await server.listen() await server.close() // Server is closed! - const tunnel = hs.tunnel(server.publicKey) + const tunnel = hs2.tunnel(server.publicKey, { keyPair }) try { await tunnel.local(await freePort(), await freePort()) @@ -302,13 +292,126 @@ test('tunnels - failed to connect to server', async function (t) { await tunnel.close() await server.close() - await hs.destroy() }) -async function createHypershell (t) { +test('chaos of tunnels', async function (t) { + t.plan(20) + + const [hs1, hs2] = await createHypershells(t) + const keyPair = crypto.keyPair() + + const server = hs1.createServer({ firewall: [keyPair.publicKey] }) + await server.listen() + + const tunnel = hs2.tunnel(server.publicKey, { keyPair }) + + // Local tunnels + const localPort1 = await freePort() + const localPort2 = await freePort() + const remotePort1 = await createTcpServer(t, function (socket) { + socket.on('data', function (data) { + socket.write('Hello World! A') + }) + }) + const remotePort2 = await createTcpServer(t, function (socket) { + socket.on('data', function (data) { + socket.write('Hello World! B') + }) + }) + + const localProxy1 = await tunnel.local(localPort1 + ':127.0.0.1', remotePort1 + ':127.0.0.1') + const localProxy2 = await tunnel.local(localPort2 + ':127.0.0.1:' + remotePort2 + ':127.0.0.1') + + // Remote tunnels + const localPort3 = await createTcpServer(t, socket => { + socket.on('data', function (data) { + socket.write('Hello World! C') + }) + }) + const localPort4 = await createTcpServer(t, socket => { + socket.on('data', function (data) { + socket.write('Hello World! D') + }) + }) + const remotePort3 = await freePort() + const remotePort4 = await freePort() + + const remoteProxy1 = await tunnel.remote(remotePort3 + ':127.0.0.1', localPort3 + ':127.0.0.1') + const remoteProxy2 = await tunnel.remote(remotePort4 + ':127.0.0.1:' + localPort4 + ':127.0.0.1') + + // Connections + t.alike(await recv(localPort1), Buffer.from('Hello World! A')) + t.alike(await recv(localPort2), Buffer.from('Hello World! B')) + t.alike(await recv(remotePort3), Buffer.from('Hello World! C')) + t.alike(await recv(remotePort4), Buffer.from('Hello World! D')) + + await localProxy1.close() + + t.alike(await recv(localPort1), null) + t.alike(await recv(localPort2), Buffer.from('Hello World! B')) + t.alike(await recv(remotePort3), Buffer.from('Hello World! C')) + t.alike(await recv(remotePort4), Buffer.from('Hello World! D')) + + await remoteProxy1.close() + + t.alike(await recv(localPort1), null) + t.alike(await recv(localPort2), Buffer.from('Hello World! B')) + t.alike(await recv(remotePort3), null) + t.alike(await recv(remotePort4), Buffer.from('Hello World! D')) + + await localProxy2.close() + + t.alike(await recv(localPort1), null) + t.alike(await recv(localPort2), null) + t.alike(await recv(remotePort3), null) + t.alike(await recv(remotePort4), Buffer.from('Hello World! D')) + + await remoteProxy2.close() + + t.alike(await recv(localPort1), null) + t.alike(await recv(localPort2), null) + t.alike(await recv(remotePort3), null) + t.alike(await recv(remotePort4), null) + + await tunnel.close() + await server.close() + + async function recv (port, host) { + const socket = net.connect(port, host || '127.0.0.1') + + socket.on('error', () => {}) + + const connecting = new Promise(resolve => socket.once('connect', () => resolve(true))) + const closing = new Promise(resolve => socket.once('close', () => resolve(false))) + + const opened = await Promise.race([connecting, closing]) + + if (!opened) { + return null + } + + socket.write('echo') + + const message = await new Promise(resolve => socket.once('data', resolve)) + + socket.end() + + await closing + + return message + } +}) + +async function createHypershells (t) { const { bootstrap } = await createTestnet(3, t.teardown) - return new Hypershell({ bootstrap }) + const a = new Hypershell({ bootstrap }) + const b = new Hypershell({ bootstrap }) + + t.teardown(() => a.destroy()) + t.teardown(() => b.destroy()) + + return [a, b] } async function createTcpServer (t, onrequest) { @@ -316,7 +419,7 @@ async function createTcpServer (t, onrequest) { t.teardown(() => new Promise(resolve => server.close(resolve))) - await listen(server, 0, '127.0.0.1') + await bind.listen(server, 0, '127.0.0.1') return server.address().port } From 355a19afb510b4b4d468db9aaa0101b12442e8de Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Fri, 18 Oct 2024 11:12:13 -0300 Subject: [PATCH 10/21] Remote tunnels reconnects on background --- lib/protocols/tunnel.js | 69 ++++++++++++++++++++++++++++++++------- test/lib.js | 71 +++++++++++++++++++++++++++++++---------- 2 files changed, 112 insertions(+), 28 deletions(-) diff --git a/lib/protocols/tunnel.js b/lib/protocols/tunnel.js index e5e42b6..caa8b0a 100644 --- a/lib/protocols/tunnel.js +++ b/lib/protocols/tunnel.js @@ -127,35 +127,68 @@ module.exports = class Tunnel { fwd.remote = tmp } - this._connect() - this._createChannel() - - // Long secret id so the server can't guess the local allow const serverId = crypto.randomBytes(32).toString('hex') - - this._allow.set(serverId, [fwd.local.host + ':' + fwd.local.port]) - - this.wireServerStart.send({ + const serverInfo = { id: serverId, port: fwd.remote.port, host: fwd.remote.host, connect: fwd.local - }) + } + + this._allow.set(serverId, [fwd.local.host + ':' + fwd.local.port]) + + this._connect() + this._createChannel() + + this.wireServerStart.send(serverInfo) const ready = this._wait(serverId) const opened = await this.channel.fullyOpened() if (!opened) { - // TODO: Check if onWireClose gets triggered if couldn't connect this._allow.delete(serverId) throw new Error('Could not connect to server') } + // Retry in case of disconnections + const closed = withResolvers() + let closing = false + let retryCount = 0 + + const onWireClose = async () => { + const wait = withResolvers() + const time = 500 * Math.min(retryCount++, 5) + const timeout = setTimeout(() => wait.resolve(), time) + + await Promise.race([wait.promise, closed.promise]) + + if (closing) { + clearTimeout(timeout) + return + } + + this._connect() + this._createChannel() + + this.wireServerStart.send(serverInfo) + } + + this._events.on('close', onWireClose) + await ready return { close: async () => { + if (closing) { + return + } + + closing = true + closed.resolve() + + this._events.off('close', onWireClose) + this._allow.delete(serverId) if (this.mux.stream.destroying || this.channel.closed) { @@ -166,9 +199,9 @@ module.exports = class Tunnel { id: serverId }) - const closed = this._wait(serverId) + const serverClosed = this._wait(serverId) - await closed + await serverClosed } } } @@ -421,3 +454,15 @@ function waitForMessage (events, id) { events.off('close', onclose) } } + +function withResolvers () { + let resolve = null + let reject = null + + const promise = new Promise((_resolve, _reject) => { + resolve = _resolve + reject = _reject + }) + + return { promise, resolve, reject } +} diff --git a/test/lib.js b/test/lib.js index 2453bcd..7588cb3 100644 --- a/test/lib.js +++ b/test/lib.js @@ -294,6 +294,45 @@ test('tunnels - failed to connect to server', async function (t) { await server.close() }) +test('tunnel - remote forwarding - retry on background', async function (t) { + t.plan(2) + + const [hs1, hs2] = await createHypershells(t) + const serverKeyPair = crypto.keyPair() + const clientKeyPair = crypto.keyPair() + + const server = hs1.createServer({ keyPair: serverKeyPair, firewall: [clientKeyPair.publicKey] }) + await server.listen() + + const tunnel = hs2.tunnel(server.publicKey, { keyPair: clientKeyPair }) + + const localPort = await createTcpServer(t, socket => { + socket.on('data', function (data) { + socket.write('Hello World!') + }) + }) + + const remotePort = await freePort() + + const proxy = await tunnel.remote(remotePort + ':127.0.0.1', localPort + ':127.0.0.1') + + t.alike(await recv(remotePort), Buffer.from('Hello World!')) + + // Server closed! + await server.close() + + const server2 = hs1.createServer({ keyPair: serverKeyPair, firewall: [clientKeyPair.publicKey] }) + await server2.listen() + + await new Promise(resolve => setTimeout(resolve, 2000)) + + t.alike(await recv(remotePort), Buffer.from('Hello World!')) + + await proxy.close() + await tunnel.close() + await server2.close() +}) + test('chaos of tunnels', async function (t) { t.plan(20) @@ -375,32 +414,32 @@ test('chaos of tunnels', async function (t) { await tunnel.close() await server.close() +}) - async function recv (port, host) { - const socket = net.connect(port, host || '127.0.0.1') +async function recv (port, host) { + const socket = net.connect(port, host || '127.0.0.1') - socket.on('error', () => {}) + socket.on('error', () => {}) - const connecting = new Promise(resolve => socket.once('connect', () => resolve(true))) - const closing = new Promise(resolve => socket.once('close', () => resolve(false))) + const connecting = new Promise(resolve => socket.once('connect', () => resolve(true))) + const closing = new Promise(resolve => socket.once('close', () => resolve(false))) - const opened = await Promise.race([connecting, closing]) + const opened = await Promise.race([connecting, closing]) - if (!opened) { - return null - } + if (!opened) { + return null + } - socket.write('echo') + socket.write('echo') - const message = await new Promise(resolve => socket.once('data', resolve)) + const message = await new Promise(resolve => socket.once('data', resolve)) - socket.end() + socket.end() - await closing + await closing - return message - } -}) + return message +} async function createHypershells (t) { const { bootstrap } = await createTestnet(3, t.teardown) From afcc23c551659fada67de08eb42d02160e215453 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Fri, 18 Oct 2024 13:42:02 -0300 Subject: [PATCH 11/21] Redo README --- README.md | 333 ++++++++++++++++++++++++++++------------- bin.js | 2 +- index.js | 4 + lib/bin/login.js | 2 +- lib/protocols/shell.js | 10 +- test/lib.js | 4 +- 6 files changed, 249 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index a3820ec..88dfdae 100644 --- a/README.md +++ b/README.md @@ -2,198 +2,329 @@ Spawn shells anywhere. Fully peer-to-peer, authenticated, and end to end encrypted. -## Install - ``` npm i -g hypershell ``` ## Usage -```shell -# Create a key -hypershell keygen [-f keyfile] [-c comment] +```sh +# Run a multi-purpose server +hypershell server # [-f keyfile] [--firewall filename] [--disable-firewall] -# Create a P2P server -hypershell server [-f keyfile] [--firewall filename] [--disable-firewall] [--protocol name] +# Shell into the server +hypershell login key-or-name # [-f keyfile] -# Connect to a P2P shell -hypershell login [-f keyfile] +# Transfer files +hypershell copy [@host:]source [@host:]target # [-f keyfile] -# Local tunnel that forwards to remote host -hypershell tunnel -L [address:]port:host:hostport +# Local and remote port forwarding +hypershell tunnel key-or-name -L [address:]port:host:hostport # [-f keyfile] +hypershell tunnel key-or-name -R [address:]port:host:hostport # [-f keyfile] -# Copy files (download and upload) -hypershell copy <[@host:]source> <[@host:]target> [-f keyfile] +# Create a key +hypershell keygen # [-f keyfile] [-c comment] ``` -Use `--help` with any command for more information, for example `hypershell server --help`. - -It can also be imported as a library: +Use `--help` with any command for more information, e.g. `hypershell server --help`. ```js const Hypershell = require('hypershell') const hs = new Hypershell() +const keyPair = Hypershell.keyPair() -const server = hs.createServer() +const server = hs.createServer({ firewall: [keyPair.publicKey] }) await server.listen() -const keyPair = crypto.keyPair() - -server.firewall = [keyPair.publicKey] +const shell = hs.login(server.publicKey, { + keyPair, + stdin: process.stdin, + stdout: process.stdout +}) -const shell = hs.login(server.publicKey, { keyPair }) +await shell.ready() +await shell.fullyClosed() -shell.stdin.write('echo "Hello World!"\n') +if (shell.exitCode !== null) { + process.exitCode = shell.exitCode +} await shell.close() await server.close() await hs.destroy() ``` -## First steps +## Server -Keys are automatically created with a default filename on first run. - -Otherwise, you can first do: +Server keys are automatically created on the first run at `~/.hypershell/id_server`. ```sh -hypershell keygen +hypershell server ``` -Just connect to servers (they have to allow your public key): +`~/.hypershell/authorized_peers` file will be empty, denying all connections by default. + +Public keys can be added to the list to allow them in real-time. + +There is a `--disable-firewall` flag to allow anyone to connect (useful for public services like game servers). + +#### Running multiple servers + +Use `-f ` to change the primary key. + +Use `--firewall ` to change the authorized peers list. ```sh -hypershell login +hypershell server -f ~/.hypershell/another_id_server --firewall ~/.hypershell/another_authorized_peers ``` -You could also create a server: +#### Change the default shell ```sh -hypershell server +SHELL=bash hypershell server ``` -`~/.hypershell/authorized_peers` file will be empty, denying all connections by default. +## Login -Public keys can be added to the list to allow them in real-time. +Client keys are automatically created on the first run at `~/.hypershell/id`. -Or you can use the `--disable-firewall` flag to allow anyone to connect, useful for public services like game servers. +Connect to a server (they have to allow your public key): -## Known peers -There will be a file `~/.hypershell/known_peers`. +```sh +hypershell login +``` + +#### Known peers -Add named peers to the file like for example: -```bash +Use the file `~/.hypershell/known_peers` to add peers by name like so: + +```sh # -home cdb7b7774c3d90547ce2038b51367dc4c96c42abf7c2e794bb5eb036ec7793cd +home nq98erpfiogzfptca3jcaum7atscfoiyu76ng9x7rfeboa9qeiat ``` -Now just `hypershell home` (it saves you writing the entire public key). +Now just `hypershell login home` (it saves you writing the entire public key). -## hypershell-copy -Similar to `scp`. It works with files, and with folders recursively. +#### Variadic command -For the next examples, `remote_peer` is a name that can be added to the `known_peers` file. +```sh +hypershell login home -- /usr/bin/bash +``` + +## Copy -Upload a file from your desktop to a remote server: -```bash -hypershell-copy ~/Desktop/file.txt @remote_peer:/root/file.txt +Upload a file or folder: + +```sh +hypershell copy ./file.txt @home:/root/uploaded.txt ``` -Download a file from a remote server to your desktop: -```bash -hypershell-copy @remote_peer:/root/database.json ~/Desktop/db-backup.json +Download a file or folder: + +```sh +hypershell copy @home:/root/database.json ./downloaded.json ``` -Note: in the future, the `@` might be removed. +#### Can use public keys + +The public key of the server can be used directly (without `@`): -You can also use the public key of the server directly (without `@`): -```bash -hypershell-copy ~/Desktop/some-folder cdb7b7774c3d90547ce2038b51367dc4c96c42abf7c2e794bb5eb036ec7793cd:/root/backup-folder +```sh +hypershell copy ./project nq98erpfiogzfptca3jcaum7atscfoiyu76ng9x7rfeboa9qeiat:/root/project ``` -## Local tunnel +## Tunnel + +#### Local port forwarding + +Create a local proxy where every connection is forwarded to the server. -#### Client +Example: Access a private service in the server but locally e.g. a database port. -It creates a local server, and every connection is forwarded to the remote host. +```sh +hypershell tunnel home -L 3000:127.0.0.1:3306:127.0.0.1 +``` + +#### Remote port forwarding -In this example, creates a local tunnel at `127.0.0.1:2020` (where you can connect to),\ -that later gets forwarded to a remote server which it connects to `127.0.0.1:3000`: -```bash -hypershell remote_peer -L 127.0.0.1:2020:127.0.0.1:3000 +Create a remote proxy where every connection is forwarded locally. -# Local 8080 -> Remote 1080 -hypershell tunnel -L 8080:127.0.0.1:1080:127.0.0.1 +Example: Expose your local development React.js app to the internet. -# Remote 80 -> Local 3000 -hypershell tunnel -R 80:127.0.0.1:3000:127.0.0.1 +```sh +hypershell tunnel home -R 80:0.0.0.0:3000:127.0.0.1 ``` -Instead of `remote_peer` you can use the server public key as well. +#### Multiple tunnels at once + +You can do this with both `-L` and `-R`. -You can also pass several `-L` to run multiple local servers that remote forwards: -```bash -hypershell remote_peer -L 2020:127.0.0.1:3000 -L 2021:127.0.0.1:3000 -L 2022:127.0.0.1:3000 +```sh +hypershell tunnel home -L 5000:5900:127.0.0.1 -L 3000:3389:127.0.0.1 ``` -#### Server +#### Restrict tunnel server -By default, `hypershell-server` runs a server with full access, including forwarding to all hosts and ports. +A server runs with full access by default, including forwarding to all hosts and ports. -You can run a server with restricted permissions to allow forwarding a specific host and port only. +You can run the server as tunnel only, and limiting to a specific set of addresses. -Let's say you have a local project like a React app at `http://127.0.0.1:3000/`,\ -you can create a restricted server to safely share this unique port like so: +Example: You want to safely share your React.js app to someone. -```bash +```sh hypershell server --protocol tunnel --tunnel 127.0.0.1:3000 ``` -Or if you want to allow multiple hosts, port range, etc: +Range of ports are also valid: `--tunnel 127.0.0.1:4100-4200` + +## API + +#### `const hs = new Hypershell([options])` + +Create a Hypershell instance. + +Available options: + +```js +{ + dht, + bootstrap +} +``` + +#### `await hs.destroy()` + +Close the Hypershell instance. + +## Server -```bash -hypershell server --protocol tunnel --tunnel 127.0.0.1:4100-4200 --tunnel 192.168.0.25:1080 +#### `const server = hs.createServer([options])` + +Create a Hypershell server. + +Available options: + +```js +{ + keyPair, + seed, + firewall: [], // Set to null to allow everyone + verbose: false, + protocols: ['shell', 'copy', 'tunnel'], + tunnel: { + // By default, allows the client to tell the server to connect to anything + allow: null // Limit it with a an array of addresses like ['127.0.0.1:3000'] + } +} ``` -Clients trying to use any different hosts/ports are automatically disconnected. +Can also edit `server.firewall = [...]` in real-time. + +## Login + +#### `const shell = hs.login(publicKey, [options])` -## Multiple keys -To have multiple servers, you need multiple keys. +Create a Shell instance. -Generate another key: -```bash -hypershell-keygen -f ~/.hypershell/my-server +Available options: + +```js +{ + keyPair, + seed, + rawArgs, + stdin, + stdout, + onerror +} ``` -Now create a new shell server: -```bash -hypershell-server -f ~/.hypershell/my-server --firewall ~/.hypershell/my-server-firewall +#### `await shell.ready()` + +Waits until is connected to the server or throws if couldn't connect. + +#### `await shell.close()` + +Close the instance. + +#### `await shell.fullyClosed()` + +Will resolve when the shell is fully closed (e.g. `exit` command). + +#### `shell.exitCode` + +Indicates the exit code from the remote shell, by default `null`. + +## Copy + +#### `const transfer = hs.copy(publicKey, [options])` + +Create a Copy instance. + +Available options: + +```js +{ + keyPair, + seed, + permissions: [], // Possible values: 'pack' and 'extract' + onerror +} +``` + +#### `await transfer.upload(source, destination)` + +Upload a file or folder. + +#### `await transfer.download(source, destination)` + +Download a file or folder. + +#### `await transfer.close()` + +Close the instance. + +## Tunnel + +#### `const tunnel = hs.tunnel(publicKey, [options])` + +Create a Tunnel instance. + +Available options: + +```js +{ + keyPair, + seed, + allow: [] // By default, it blocks all remote connect commands +} ``` -The client also accepts `-f` in case you need it. +#### `const proxy = await tunnel.local(localAddress, remoteAddress)` -## Restrict server protocols +Create a local proxy server that forwards to the remote address. -This is the list of server protocols: -- `shell` -- `upload` -- `download` -- `tunnel` +Throws an error if initially can't connect to the server. -By default, all of them are enabled when running a server. +In case of disconnections, it automatically recovers on the next local connection. -For example, you could limit it to shell only:\ -`hypershell-server --protocol shell` +#### `const proxy = await tunnel.remote(remoteAddress, localAddress)` -Or only allow file upload and/or download:\ -`hypershell-server --protocol upload --protocol download` +Create a proxy on the server that forwards to the local address. -Restrict to tunnel only:\ -`hypershell-server --protocol tunnel` +Throws an error if initially can't connect to the server. -For example, if you only allow `tunnel`, then any attempt from clients to `shell` into the server will auto disconnect them. +In case of disconnections, it reconnects on background and resends the remote server command. + +#### `await proxy.close()` + +Stop the proxy. + +#### `await tunnel.close()` + +Close the instance. ## License + Apache-2.0 diff --git a/bin.js b/bin.js index 4ffc9ac..3e64ac3 100755 --- a/bin.js +++ b/bin.js @@ -7,11 +7,11 @@ const pkg = require('./package.json') const main = program .version(pkg.version) .description(pkg.description) - .addCommand(require('./bin/keygen.js')) .addCommand(require('./bin/server.js')) .addCommand(require('./bin/login.js')) .addCommand(require('./bin/copy.js')) .addCommand(require('./bin/tunnel.js')) + .addCommand(require('./bin/keygen.js')) main.parseAsync().catch(err => { safetyCatch(err) diff --git a/index.js b/index.js index ad71e44..7a9a69e 100644 --- a/index.js +++ b/index.js @@ -79,6 +79,10 @@ module.exports = class Hypershell { await this.dht.destroy() } } + + static keyPair (seed) { + return crypto.keyPair(seed) + } } class Server { diff --git a/lib/bin/login.js b/lib/bin/login.js index e4f651f..bfbecaf 100644 --- a/lib/bin/login.js +++ b/lib/bin/login.js @@ -35,7 +35,7 @@ module.exports = async function login (keyOrName, opts = {}) { } }) - await shell.channel.fullyClosed() + await shell.fullyClosed() if (shell.exitCode !== null) { process.exitCode = shell.exitCode diff --git a/lib/protocols/shell.js b/lib/protocols/shell.js index 03957fe..c46e4ee 100644 --- a/lib/protocols/shell.js +++ b/lib/protocols/shell.js @@ -122,7 +122,11 @@ class ShellClient { } async ready () { - await this.channel.fullyOpened() + const opened = await this.channel.fullyOpened() + + if (!opened) { + throw new Error('Could not connect to server') + } } async close () { @@ -133,6 +137,10 @@ class ShellClient { this.socket.destroy() } + async fullyClosed () { + await this.channel.fullyClosed() + } + onWireClose () { this.socket.destroy() } diff --git a/test/lib.js b/test/lib.js index 7588cb3..efabdf0 100644 --- a/test/lib.js +++ b/test/lib.js @@ -58,7 +58,7 @@ test('shell - server is closed first', async function (t) { await shell.ready() await server.close() - await shell.channel.fullyClosed() + await shell.fullyClosed() const err = getStreamError(shell.socket) @@ -82,7 +82,7 @@ test('shell - exit code', async function (t) { shell.stdin.write('exit 127\n') - await shell.channel.fullyClosed() + await shell.fullyClosed() t.is(shell.exitCode, 127) await server.close() From 49f4cbaa59c163451010f21494abe739b97b8e82 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Fri, 18 Oct 2024 17:59:21 -0300 Subject: [PATCH 12/21] Add hs.admin() for short seed invites --- README.md | 52 +++++++++++++++++++- bin/login.js | 1 + index.js | 44 +++++++++++++++-- lib/bin/login.js | 30 +++++++++++- lib/protocols/admin.js | 99 +++++++++++++++++++++++++++++++++++++++ lib/protocols/messages.js | 29 ++++++++++++ package.json | 3 +- test/lib.js | 35 +++++++++++++- 8 files changed, 286 insertions(+), 7 deletions(-) create mode 100644 lib/protocols/admin.js diff --git a/README.md b/README.md index 88dfdae..e93d34c 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,23 @@ Now just `hypershell login home` (it saves you writing the entire public key). hypershell login home -- /usr/bin/bash ``` +#### Invite + +Create a short seed for someone to join once into your server: + +```sh +hypershell login home --invite +# One time invite: hkwsesi4dm1ng +``` + +Then someone can use it only once to log in: + +```sh +hypershell login home --invite hkwsesi4dm1ng +``` + +They can add themselves into `~/.hypershell/authorized_peers` for permanent access. + ## Copy Upload a file or folder: @@ -211,7 +228,7 @@ Available options: seed, firewall: [], // Set to null to allow everyone verbose: false, - protocols: ['shell', 'copy', 'tunnel'], + protocols: ['shell', 'copy', 'tunnel', 'admin'], tunnel: { // By default, allows the client to tell the server to connect to anything allow: null // Limit it with a an array of addresses like ['127.0.0.1:3000'] @@ -325,6 +342,39 @@ Stop the proxy. Close the instance. +## Admin + +#### `const admin = hs.admin(publicKey, [options])` + +Create an Admin instance. + +Available options: + +```js +{ + keyPair, + seed +} +``` + +#### `const shortSeed = await admin.createInvite([options])` + +Returns an 8-byte seed to be used to make a key pair. + +The public key derived from this short seed is only allowed once in the firewall. + +Available options: + +```js +{ + expiry: 60 * 60 * 1000 +} +``` + +#### `await admin.close()` + +Close the instance. + ## License Apache-2.0 diff --git a/bin/login.js b/bin/login.js index 33a1886..538d36e 100644 --- a/bin/login.js +++ b/bin/login.js @@ -3,6 +3,7 @@ const { createCommand } = require('commander') module.exports = createCommand('login') .description('connect to a P2P shell') .argument('', 'public key or name of the server') + .option('--invite [token]', 'create or use a token to join the server') .option('-f ', 'filename of the client seed key') .option('--bootstrap ', 'custom dht nodes') .action(require('../lib/bin/login.js')) diff --git a/index.js b/index.js index 7a9a69e..d4c38fb 100644 --- a/index.js +++ b/index.js @@ -6,6 +6,7 @@ const safetyCatch = require('safety-catch') const { ShellServer, ShellClient } = require('./lib/protocols/shell.js') const Copy = require('./lib/protocols/copy.js') const Tunnel = require('./lib/protocols/tunnel.js') +const { AdminServer, AdminClient } = require('./lib/protocols/admin.js') module.exports = class Hypershell { constructor (opts = {}) { @@ -15,7 +16,9 @@ module.exports = class Hypershell { } createServer (opts = {}) { - return new Server(this.dht, { ...opts, onsocket }) + const server = new Server(this.dht, { ...opts, onsocket }) + + return server function onsocket (socket) { const mux = Protomux.from(socket) @@ -40,6 +43,12 @@ module.exports = class Hypershell { }) }) } + + if (this.protocols.includes('admin')) { + mux.pair({ protocol: 'hypershell-admin' }, () => { + AdminServer.attach(socket, server) + }) + } } } @@ -74,6 +83,10 @@ module.exports = class Hypershell { return new Tunnel(this.dht, publicKey, opts) } + admin (publicKey, opts = {}) { + return new AdminClient(this.dht, publicKey, opts) + } + async destroy () { if (this._autoDestroy) { await this.dht.destroy() @@ -88,10 +101,13 @@ module.exports = class Hypershell { class Server { constructor (dht, opts = {}) { this.dht = dht + this.keyPair = opts.keyPair || crypto.keyPair(opts.seed) this.firewall = opts.firewall || opts.firewall === null ? opts.firewall : [] this.verbose = !!opts.verbose - this.protocols = opts.protocols || ['shell', 'copy', 'tunnel'] + this.protocols = opts.protocols || ['shell', 'copy', 'tunnel', 'admin'] + + this.invites = new Map() this._server = this.dht.createServer({ firewall: this._onFirewall.bind(this) @@ -142,10 +158,24 @@ class Server { } _onFirewall (remotePublicKey, remoteHandshakePayload) { + this._cleanupInvites() + if (this.firewall === null) { return false } + for (const [publicKey] of this.invites) { + if (remotePublicKey.equals(Buffer.from(publicKey, 'hex'))) { + if (this.verbose) { + console.log('Invite accepted:', HypercoreId.encode(remotePublicKey)) + } + + this.invites.delete(publicKey) + + return false + } + } + for (const publicKey of this.firewall) { if (remotePublicKey.equals(publicKey)) { return false @@ -153,11 +183,19 @@ class Server { } if (this.verbose) { - console.log('Firewall denied', HypercoreId.encode(remotePublicKey)) + console.log('Firewall denied:', HypercoreId.encode(remotePublicKey)) } return true } + + _cleanupInvites () { + for (const [publicKey, expiry] of this.invites) { + if (expiry - Date.now() < 0) { + this.invites.delete(publicKey) + } + } + } } function closeConnections (sockets, force) { diff --git a/lib/bin/login.js b/lib/bin/login.js index bfbecaf..c643be2 100644 --- a/lib/bin/login.js +++ b/lib/bin/login.js @@ -1,5 +1,7 @@ const fs = require('fs') const path = require('path') +const crypto = require('hypercore-crypto') +const z32 = require('z32') const constants = require('../constants.js') const keygen = require('./keygen.js') const Hypershell = require('../../index.js') @@ -19,8 +21,34 @@ module.exports = async function login (keyOrName, opts = {}) { bootstrap: opts.bootstrap }) + if (opts.invite === true) { + const admin = hs.admin(serverPublicKey, { + keyPair: await fileToKeyPair(keyFilename) + }) + + const invite = await admin.createInvite() + + console.log('One time invite:', z32.encode(invite)) + + await admin.close() + await hs.destroy() + + return + } + + let keyPair = null + + if (typeof opts.invite === 'string') { + const shortSeed = z32.decode(opts.invite) + const seed = Buffer.alloc(32).fill(shortSeed, 0, shortSeed.length) + + keyPair = crypto.keyPair(seed) + } else { + keyPair = await fileToKeyPair(keyFilename) + } + const shell = hs.login(serverPublicKey, { - keyPair: await fileToKeyPair(keyFilename), + keyPair, rawArgs: this.rawArgs, stdin: process.stdin, stdout: process.stdout, diff --git a/lib/protocols/admin.js b/lib/protocols/admin.js new file mode 100644 index 0000000..6156ac2 --- /dev/null +++ b/lib/protocols/admin.js @@ -0,0 +1,99 @@ +const crypto = require('hypercore-crypto') +const ProtomuxRPC = require('protomux-rpc') +const m = require('./messages.js') + +class AdminServer { + constructor (socket, server) { + this.server = server + + this.rpc = new ProtomuxRPC(socket, { + protocol: 'hypershell-admin' + }) + + this.rpc.respond('invite', m.admin.invite, this.onWireInvite.bind(this)) + } + + static attach (socket, server) { + return new this(socket, server) + } + + onWireInvite (req, b) { + this.server._cleanupInvites() + + const shortSeed = crypto.randomBytes(8) + const seed = Buffer.alloc(32).fill(shortSeed, 0, shortSeed.length) + const keyPair = crypto.keyPair(seed) + + const expiration = Date.now() + (req.expiry || 60 * 60 * 1000) + + this.server.invites.set(keyPair.publicKey.toString('hex'), expiration) + + return shortSeed + } +} + +class AdminClient { + constructor (dht, publicKey, opts = {}) { + this.dht = dht + this.publicKey = publicKey + + this.keyPair = opts.keyPair || crypto.keyPair(opts.seed) + + this.rpc = null + } + + _connect () { + if (this.rpc && !this.rpc.stream.destroying) { + return + } + + const socket = this.dht.connect(this.publicKey, { + keyPair: this.keyPair, + reusableSocket: true + }) + + socket.setKeepAlive(5000) + + this.rpc = new ProtomuxRPC(socket, { + protocol: 'hypershell-admin' + }) + } + + static attach (mux) { + return new this(mux) + } + + async ready () { + this._connect() + + // TODO + const opened = await this.rpc._channel.fullyOpened() + + if (!opened) { + throw new Error('Could not connect to server') + } + } + + async close () { + this.rpc.destroy() + + await this.rpc._channel.fullyClosed() + + this.rpc.stream.destroy() + } + + async createInvite (opts = {}) { + await this.ready() + + const shortSeed = await this.rpc.request('invite', { + expiry: opts.expiry || 0 + }, m.admin.invite) + + return shortSeed + } +} + +module.exports = { + AdminServer, + AdminClient +} diff --git a/lib/protocols/messages.js b/lib/protocols/messages.js index 80a29f1..31c3133 100644 --- a/lib/protocols/messages.js +++ b/lib/protocols/messages.js @@ -191,3 +191,32 @@ tunnel.pump = { } } } + +const admin = exports.admin = {} + +admin.invite = { + requestEncoding: { + preencode (state, v) { + c.uint.preencode(state, v.expiry) + }, + encode (state, v) { + c.uint.encode(state, v.expiry) + }, + decode (state) { + return { + expiry: c.uint.decode(state) + } + } + }, + responseEncoding: { + preencode (state, v) { + c.buffer.preencode(state, v) + }, + encode (state, v) { + c.buffer.encode(state, v) + }, + decode (state) { + return c.buffer.decode(state) + } + } +} diff --git a/package.json b/package.json index 1888a4c..83a6ad9 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "read-file-live": "^1.0.1", "tar-fs": "^3.0.6", "tiny-configs": "^1.1.0", - "tt-native": "^1.1.1" + "tt-native": "^1.1.1", + "z32": "^1.1.0" } } diff --git a/test/lib.js b/test/lib.js index efabdf0..e502dcb 100644 --- a/test/lib.js +++ b/test/lib.js @@ -416,6 +416,37 @@ test('chaos of tunnels', async function (t) { await server.close() }) +test('invite', async function (t) { + t.plan(1) + + const [hs1, hs2, hs3] = await createHypershells(t) + const keyPair = crypto.keyPair() + + const server = hs1.createServer({ firewall: [keyPair.publicKey] }) + await server.listen() + + const admin = hs2.admin(server.publicKey, { keyPair }) + const invite = await admin.createInvite() + await admin.close() + + const seed = Buffer.alloc(32).fill(invite, 0, invite.length) + const keyPairInvite = crypto.keyPair(seed) + + const shell = hs3.login(server.publicKey, { keyPair: keyPairInvite }) + await shell.ready() // Logged in + await shell.close() + + try { + const shell = hs3.login(server.publicKey, { keyPair: keyPairInvite }) + await shell.ready() + t.fail() + } catch (err) { + t.is(err.message, 'Could not connect to server') + } + + await server.close() +}) + async function recv (port, host) { const socket = net.connect(port, host || '127.0.0.1') @@ -446,11 +477,13 @@ async function createHypershells (t) { const a = new Hypershell({ bootstrap }) const b = new Hypershell({ bootstrap }) + const c = new Hypershell({ bootstrap }) t.teardown(() => a.destroy()) t.teardown(() => b.destroy()) + t.teardown(() => c.destroy()) - return [a, b] + return [a, b, c] } async function createTcpServer (t, onrequest) { From 630965ea2a3bc0f191e30c93d415c822658610fb Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Fri, 18 Oct 2024 20:40:31 -0300 Subject: [PATCH 13/21] Fix test for Windows --- test/lib.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib.js b/test/lib.js index e502dcb..86be0bc 100644 --- a/test/lib.js +++ b/test/lib.js @@ -36,7 +36,7 @@ test('basic shell', async function (t) { } }) - shell.stdin.write('echo "Hello World!"\n') + shell.stdin.write('echo "Hello World!"\r\n') await t2 @@ -80,7 +80,7 @@ test('shell - exit code', async function (t) { const shell = hs2.login(server.publicKey, { keyPair }) await shell.ready() - shell.stdin.write('exit 127\n') + shell.stdin.write('exit 127\r\n') await shell.fullyClosed() t.is(shell.exitCode, 127) From bbe5391637f607ddafbd27db76ea83d14dbb8fa0 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Sat, 19 Oct 2024 21:33:43 -0300 Subject: [PATCH 14/21] Multiple tunnels at once --- lib/bin/tunnel.js | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/bin/tunnel.js b/lib/bin/tunnel.js index dbfa892..8bc09ff 100644 --- a/lib/bin/tunnel.js +++ b/lib/bin/tunnel.js @@ -14,6 +14,10 @@ module.exports = async function tunnel (keyOrName, opts = {}) { await keygen({ f: keyFilename }) } + if (!opts.L && !opts.R) { + throw new Error('-L o -R is required') + } + const serverPublicKey = await getKnownPeer(keyOrName, { verbose: true }) const hs = new Hypershell({ bootstrap: opts.bootstrap }) @@ -22,14 +26,18 @@ module.exports = async function tunnel (keyOrName, opts = {}) { keyPair: await fileToKeyPair(keyFilename) }) - let proxy = null + const proxies = [] if (opts.L) { - proxy = await tunnel.local(opts.L) - } else if (opts.R) { - proxy = await tunnel.remote(opts.R) - } else { - throw new Error('-L o -R is required') + for (const local of opts.L) { + proxies.push(await tunnel.local(local)) + } + } + + if (opts.R) { + for (const remote of opts.R) { + proxies.push(await tunnel.remote(remote)) + } } console.log('Tunnel is ready!') @@ -43,7 +51,9 @@ module.exports = async function tunnel (keyOrName, opts = {}) { } async function close () { - await proxy.close() + for (const proxy of proxies) { + await proxy.close() + } await tunnel.close() await hs.destroy() } From d53af5b3b307388fdf0b7f59973ec2c52b42804a Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Sun, 20 Oct 2024 02:40:14 -0300 Subject: [PATCH 15/21] Improve tunnels code with ReadyResource --- README.md | 12 +- lib/bin/tunnel.js | 8 +- lib/protocols/tunnel.js | 283 ++++++++++++++++++++++++---------------- package.json | 1 + test/lib.js | 38 ++++-- 5 files changed, 214 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index e93d34c..dc4f172 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,11 @@ Available options: } ``` -#### `const proxy = await tunnel.local(localAddress, remoteAddress)` +#### `await tunnel.ready()` + +Optionally wait for being connected to the server. + +#### `const proxy = tunnel.local(localAddress, remoteAddress)` Create a local proxy server that forwards to the remote address. @@ -326,7 +330,7 @@ Throws an error if initially can't connect to the server. In case of disconnections, it automatically recovers on the next local connection. -#### `const proxy = await tunnel.remote(remoteAddress, localAddress)` +#### `const proxy = tunnel.remote(remoteAddress, localAddress)` Create a proxy on the server that forwards to the local address. @@ -334,6 +338,10 @@ Throws an error if initially can't connect to the server. In case of disconnections, it reconnects on background and resends the remote server command. +#### `await proxy.ready()` + +Wait for the proxy to be listening for connections. + #### `await proxy.close()` Stop the proxy. diff --git a/lib/bin/tunnel.js b/lib/bin/tunnel.js index 8bc09ff..531ec28 100644 --- a/lib/bin/tunnel.js +++ b/lib/bin/tunnel.js @@ -30,16 +30,20 @@ module.exports = async function tunnel (keyOrName, opts = {}) { if (opts.L) { for (const local of opts.L) { - proxies.push(await tunnel.local(local)) + proxies.push(tunnel.local(local)) } } if (opts.R) { for (const remote of opts.R) { - proxies.push(await tunnel.remote(remote)) + proxies.push(tunnel.remote(remote)) } } + for (const proxy of proxies) { + await proxy.ready() + } + console.log('Tunnel is ready!') const unregister = goodbye(close) diff --git a/lib/protocols/tunnel.js b/lib/protocols/tunnel.js index caa8b0a..9706326 100644 --- a/lib/protocols/tunnel.js +++ b/lib/protocols/tunnel.js @@ -7,11 +7,160 @@ const DHT = require('hyperdht') const Protomux = require('protomux') const SecretStream = require('@hyperswarm/secret-stream') const bind = require('like-bind') +const ReadyResource = require('ready-resource') const safetyCatch = require('safety-catch') const m = require('./messages.js') -module.exports = class Tunnel { +class LocalPortForwarding extends ReadyResource { + constructor (tunnel, localAddress, remoteAddress, opts = {}) { + super() + + this._tunnel = tunnel + + this._localAddress = localAddress + this._fwd = parseForwardFormat(localAddress + (remoteAddress ? ':' + remoteAddress : '')) + + this._serverId = opts.serverId || null + + this._server = null + + this.ready().catch(safetyCatch) + } + + async _open () { + if (this._tunnel.closing) { + throw new Error('Tunnel is closed') + } + + // Early connect for faster initial connections also + this._tunnel._connect() + this._tunnel._createChannel() + + const opened = await this._tunnel.channel.fullyOpened() + + if (!opened) { + throw new Error('Could not connect to server') + } + + this._server = net.createServer(localSocket => { + this._tunnel._onLocalConnection(this._fwd, localSocket, this._serverId) + }) + + await bind.listen(this._server, this._fwd.local.port, this._fwd.local.host) + } + + async _close () { + await bind.close(this._server, { force: true }) + } + + forwardTo (remoteAddress) { + this._fwd = parseForwardFormat(this._localAddress + ':' + remoteAddress) + } +} + +class RemotePortForwarding extends ReadyResource { + constructor (tunnel, remoteAddress, localAddress) { + super() + + this._tunnel = tunnel + + const fwd = parseForwardFormat((localAddress ? localAddress + ':' : '') + remoteAddress) + + // TODO: Maybe pass two args to parseForwardFormat or inverse option + if (!localAddress) { + const tmp = fwd.local + + fwd.local = fwd.remote + fwd.remote = tmp + } + + this._fwd = fwd + + this._serverId = crypto.randomBytes(32).toString('hex') + this._serverInfo = { + id: this._serverId, + port: fwd.remote.port, + host: fwd.remote.host, + connect: fwd.local + } + + this._tunnel._allow.set(this._serverId, [fwd.local.host + ':' + fwd.local.port]) + + this._signalClose = withResolvers() + this._retryCount = 0 + this._onWireCloseBound = this._onWireClose.bind(this) + + this.ready().catch(safetyCatch) + } + + async _open () { + if (this._tunnel.closing) { + throw new Error('Tunnel is closed') + } + + this._tunnel._connect() + this._tunnel._createChannel() + + this._tunnel.wireServerStart.send(this._serverInfo) + + const ready = this._tunnel._wait(this._serverId) + const opened = await this._tunnel.channel.fullyOpened() + + if (!opened) { + this._tunnel._allow.delete(this._serverId) + + throw new Error('Could not connect to server') + } + + // Retry in case of disconnections + this._tunnel._events.on('close', this._onWireCloseBound) + + await ready + } + + async _close () { + this._tunnel._events.off('close', this._onWireCloseBound) + + this._tunnel._allow.delete(this._serverId) + + this._signalClose.resolve() + + if (this._tunnel.mux.stream.destroying || this._tunnel.channel.closed) { + return + } + + this._tunnel.wireServerClose.send({ + id: this._serverId + }) + + const serverClosed = this._tunnel._wait(this._serverId) + + await serverClosed + } + + async _onWireClose () { + const wait = withResolvers() + const time = 500 * Math.min(this._retryCount++, 5) + const timeout = setTimeout(() => wait.resolve(), time) + + await Promise.race([wait.promise, this._signalClose.promise]) + + if (this.closing || this._tunnel.closing) { + clearTimeout(timeout) + return + } + + this._tunnel._connect() + this._tunnel._createChannel() + + this._tunnel.wireServerStart.send(this._serverInfo) + } +} + +module.exports = class Tunnel extends ReadyResource { constructor (dht, publicKey, opts = {}) { + super() + this.dht = dht this.publicKey = publicKey @@ -87,126 +236,26 @@ module.exports = class Tunnel { tunnel._createChannel() } - async local (localAddress, remoteAddress, opts = {}) { - let fwd = parseForwardFormat(localAddress + (remoteAddress ? ':' + remoteAddress : '')) - - // Early connect for faster initial connections also - this._connect() - this._createChannel() - - const opened = await this.channel.fullyOpened() - - if (!opened) { - throw new Error('Could not connect to server') - } - - const server = net.createServer(localSocket => { - this._onLocalConnection(fwd, localSocket, opts.serverId) - }) - - await bind.listen(server, fwd.local.port, fwd.local.host) - - return { - forwardTo: function (remoteAddress) { - fwd = parseForwardFormat(localAddress + ':' + remoteAddress) - }, - close: async function () { - await bind.close(server, { force: true }) - } - } + local (localAddress, remoteAddress, opts = {}) { + return new LocalPortForwarding(this, localAddress, remoteAddress, opts) } - async remote (remoteAddress, localAddress) { - const fwd = parseForwardFormat((localAddress ? localAddress + ':' : '') + remoteAddress) - - // TODO: Maybe pass two args to parseForwardFormat - if (!localAddress) { - const tmp = fwd.local - - fwd.local = fwd.remote - fwd.remote = tmp - } - - const serverId = crypto.randomBytes(32).toString('hex') - const serverInfo = { - id: serverId, - port: fwd.remote.port, - host: fwd.remote.host, - connect: fwd.local - } - - this._allow.set(serverId, [fwd.local.host + ':' + fwd.local.port]) + remote (remoteAddress, localAddress) { + return new RemotePortForwarding(this, remoteAddress, localAddress) + } + async _open () { this._connect() this._createChannel() - this.wireServerStart.send(serverInfo) - - const ready = this._wait(serverId) const opened = await this.channel.fullyOpened() if (!opened) { - this._allow.delete(serverId) - throw new Error('Could not connect to server') } - - // Retry in case of disconnections - const closed = withResolvers() - let closing = false - let retryCount = 0 - - const onWireClose = async () => { - const wait = withResolvers() - const time = 500 * Math.min(retryCount++, 5) - const timeout = setTimeout(() => wait.resolve(), time) - - await Promise.race([wait.promise, closed.promise]) - - if (closing) { - clearTimeout(timeout) - return - } - - this._connect() - this._createChannel() - - this.wireServerStart.send(serverInfo) - } - - this._events.on('close', onWireClose) - - await ready - - return { - close: async () => { - if (closing) { - return - } - - closing = true - closed.resolve() - - this._events.off('close', onWireClose) - - this._allow.delete(serverId) - - if (this.mux.stream.destroying || this.channel.closed) { - return - } - - this.wireServerClose.send({ - id: serverId - }) - - const serverClosed = this._wait(serverId) - - await serverClosed - } - } } - async close () { + async _close () { if (this.mux) { this.mux.destroy() @@ -233,15 +282,21 @@ module.exports = class Tunnel { stream.destroy() } - for (const [, proxy] of this.proxies) { + this.streams.clear() + + for (const [id, proxy] of this.proxies) { + this.proxies.delete(id) + await proxy.close().catch(safetyCatch) } - - this.streams.clear() - this.proxies.clear() } _onLocalConnection (fwd, localSocket, serverId) { + if (this.closing) { + localSocket.destroy() + return + } + this._connect() this._createChannel() @@ -267,7 +322,9 @@ module.exports = class Tunnel { async onWireServerStart (data, c) { const { id, port, host, connect } = data - const proxy = await this.local(port + ':' + host, connect.port + ':' + connect.host, { serverId: id }) + const proxy = this.local(port + ':' + host, connect.port + ':' + connect.host, { serverId: id }) + + await proxy.ready() if (c.closed) { await proxy.close().catch(safetyCatch) diff --git a/package.json b/package.json index 83a6ad9..9a694c7 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "protomux-rpc": "^1.6.0", "pump": "^3.0.2", "read-file-live": "^1.0.1", + "ready-resource": "^1.1.1", "tar-fs": "^3.0.6", "tiny-configs": "^1.1.0", "tt-native": "^1.1.1", diff --git a/test/lib.js b/test/lib.js index 86be0bc..62a9c1f 100644 --- a/test/lib.js +++ b/test/lib.js @@ -152,7 +152,9 @@ test('basic tunnel - local forwarding', async function (t) { }) }) - const proxy1 = await tunnel.local(localPort + ':127.0.0.1', remotePort + ':127.0.0.1') + const proxy = tunnel.local(localPort + ':127.0.0.1', remotePort + ':127.0.0.1') + await proxy.ready() + const socket = net.connect(localPort, '127.0.0.1') socket.on('data', function (data) { @@ -165,7 +167,7 @@ test('basic tunnel - local forwarding', async function (t) { socket.end() await new Promise(resolve => socket.on('close', resolve)) - await proxy1.close() + await proxy.close() await tunnel.close() await server.close() }) @@ -199,7 +201,8 @@ test('tunnel allowance', async function (t) { const tunnel = hs2.tunnel(server.publicKey, { keyPair }) - const proxy = await tunnel.local(localPort + ':127.0.0.1', blockedRemotePort + ':127.0.0.1') + const proxy = tunnel.local(localPort + ':127.0.0.1', blockedRemotePort + ':127.0.0.1') + await proxy.ready() // Fails to connect const socket = net.connect(localPort, '127.0.0.1') @@ -245,7 +248,8 @@ test('basic tunnel - remote forwarding', async function (t) { const remotePort = await freePort() - const proxy = await tunnel.remote(remotePort + ':127.0.0.1', localPort + ':127.0.0.1') + const proxy = tunnel.remote(remotePort + ':127.0.0.1', localPort + ':127.0.0.1') + await proxy.ready() const socket = net.connect(remotePort, '127.0.0.1') @@ -277,14 +281,18 @@ test('tunnels - failed to connect to server', async function (t) { const tunnel = hs2.tunnel(server.publicKey, { keyPair }) try { - await tunnel.local(await freePort(), await freePort()) + const proxy = tunnel.local(await freePort(), await freePort()) + await proxy.ready() + t.fail() } catch (err) { t.is(err.message, 'Could not connect to server') } try { - await tunnel.remote(await freePort(), await freePort()) + const proxy = tunnel.remote(await freePort(), await freePort()) + await proxy.ready() + t.fail() } catch (err) { t.is(err.message, 'Could not connect to server') @@ -314,7 +322,8 @@ test('tunnel - remote forwarding - retry on background', async function (t) { const remotePort = await freePort() - const proxy = await tunnel.remote(remotePort + ':127.0.0.1', localPort + ':127.0.0.1') + const proxy = tunnel.remote(remotePort + ':127.0.0.1', localPort + ':127.0.0.1') + await proxy.ready() t.alike(await recv(remotePort), Buffer.from('Hello World!')) @@ -343,6 +352,7 @@ test('chaos of tunnels', async function (t) { await server.listen() const tunnel = hs2.tunnel(server.publicKey, { keyPair }) + await tunnel.ready() // Local tunnels const localPort1 = await freePort() @@ -358,8 +368,11 @@ test('chaos of tunnels', async function (t) { }) }) - const localProxy1 = await tunnel.local(localPort1 + ':127.0.0.1', remotePort1 + ':127.0.0.1') - const localProxy2 = await tunnel.local(localPort2 + ':127.0.0.1:' + remotePort2 + ':127.0.0.1') + const localProxy1 = tunnel.local(localPort1 + ':127.0.0.1', remotePort1 + ':127.0.0.1') + const localProxy2 = tunnel.local(localPort2 + ':127.0.0.1:' + remotePort2 + ':127.0.0.1') + + await localProxy1.ready() + await localProxy2.ready() // Remote tunnels const localPort3 = await createTcpServer(t, socket => { @@ -375,8 +388,11 @@ test('chaos of tunnels', async function (t) { const remotePort3 = await freePort() const remotePort4 = await freePort() - const remoteProxy1 = await tunnel.remote(remotePort3 + ':127.0.0.1', localPort3 + ':127.0.0.1') - const remoteProxy2 = await tunnel.remote(remotePort4 + ':127.0.0.1:' + localPort4 + ':127.0.0.1') + const remoteProxy1 = tunnel.remote(remotePort3 + ':127.0.0.1', localPort3 + ':127.0.0.1') + const remoteProxy2 = tunnel.remote(remotePort4 + ':127.0.0.1:' + localPort4 + ':127.0.0.1') + + await remoteProxy1.ready() + await remoteProxy2.ready() // Connections t.alike(await recv(localPort1), Buffer.from('Hello World! A')) From 8b0a39591e30039724b6be7b6b3297b9463590b4 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Sun, 20 Oct 2024 03:33:03 -0300 Subject: [PATCH 16/21] Inverse tunnel format --- lib/protocols/tunnel.js | 18 +++++++++--------- test/lib.js | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/protocols/tunnel.js b/lib/protocols/tunnel.js index 9706326..f64d685 100644 --- a/lib/protocols/tunnel.js +++ b/lib/protocols/tunnel.js @@ -322,7 +322,7 @@ module.exports = class Tunnel extends ReadyResource { async onWireServerStart (data, c) { const { id, port, host, connect } = data - const proxy = this.local(port + ':' + host, connect.port + ':' + connect.host, { serverId: id }) + const proxy = this.local(host + ':' + port, connect.host + ':' + connect.port, { serverId: id }) await proxy.ready() @@ -418,20 +418,20 @@ module.exports = class Tunnel extends ReadyResource { } function parseForwardFormat (value) { - const local = { port: null, host: null } - const remote = { port: null, host: null } + const local = { host: null, port: null } + const remote = { host: null, port: null } for (const part of value.split(':')) { - const isNumber = !isNaN(part) + const isHost = isNaN(part) - if (isNumber) { + if (isHost) { + if (!local.port) local.host = part + else if (!remote.port) remote.host = part + else throw new Error('Invalid host format') + } else { if (!local.port) local.port = parseInt(part, 10) else if (!remote.port) remote.port = parseInt(part, 10) else throw new Error('Invalid port format') - } else { - if (remote.port) remote.host = part - else if (local.port) local.host = part - else throw new Error('Invalid address format') } } diff --git a/test/lib.js b/test/lib.js index 62a9c1f..7b794eb 100644 --- a/test/lib.js +++ b/test/lib.js @@ -152,7 +152,7 @@ test('basic tunnel - local forwarding', async function (t) { }) }) - const proxy = tunnel.local(localPort + ':127.0.0.1', remotePort + ':127.0.0.1') + const proxy = tunnel.local('127.0.0.1:' + localPort, '127.0.0.1:' + remotePort) await proxy.ready() const socket = net.connect(localPort, '127.0.0.1') @@ -201,7 +201,7 @@ test('tunnel allowance', async function (t) { const tunnel = hs2.tunnel(server.publicKey, { keyPair }) - const proxy = tunnel.local(localPort + ':127.0.0.1', blockedRemotePort + ':127.0.0.1') + const proxy = tunnel.local('127.0.0.1:' + localPort, '127.0.0.1:' + blockedRemotePort) await proxy.ready() // Fails to connect @@ -209,7 +209,7 @@ test('tunnel allowance', async function (t) { await new Promise(resolve => socket.on('close', resolve)) // Change to the allowed remote port - proxy.forwardTo(remotePort + ':127.0.0.1') + proxy.forwardTo('127.0.0.1:' + remotePort) // Able to connect const socket2 = net.connect(localPort, '127.0.0.1') @@ -248,7 +248,7 @@ test('basic tunnel - remote forwarding', async function (t) { const remotePort = await freePort() - const proxy = tunnel.remote(remotePort + ':127.0.0.1', localPort + ':127.0.0.1') + const proxy = tunnel.remote('127.0.0.1:' + remotePort, '127.0.0.1:' + localPort) await proxy.ready() const socket = net.connect(remotePort, '127.0.0.1') @@ -322,7 +322,7 @@ test('tunnel - remote forwarding - retry on background', async function (t) { const remotePort = await freePort() - const proxy = tunnel.remote(remotePort + ':127.0.0.1', localPort + ':127.0.0.1') + const proxy = tunnel.remote('127.0.0.1:' + remotePort, '127.0.0.1:' + localPort) await proxy.ready() t.alike(await recv(remotePort), Buffer.from('Hello World!')) @@ -368,8 +368,8 @@ test('chaos of tunnels', async function (t) { }) }) - const localProxy1 = tunnel.local(localPort1 + ':127.0.0.1', remotePort1 + ':127.0.0.1') - const localProxy2 = tunnel.local(localPort2 + ':127.0.0.1:' + remotePort2 + ':127.0.0.1') + const localProxy1 = tunnel.local('127.0.0.1:' + localPort1, '127.0.0.1:' + remotePort1) + const localProxy2 = tunnel.local('127.0.0.1: ' + localPort2 + ':127.0.0.1:' + remotePort2) await localProxy1.ready() await localProxy2.ready() @@ -388,8 +388,8 @@ test('chaos of tunnels', async function (t) { const remotePort3 = await freePort() const remotePort4 = await freePort() - const remoteProxy1 = tunnel.remote(remotePort3 + ':127.0.0.1', localPort3 + ':127.0.0.1') - const remoteProxy2 = tunnel.remote(remotePort4 + ':127.0.0.1:' + localPort4 + ':127.0.0.1') + const remoteProxy1 = tunnel.remote('127.0.0.1:' + remotePort3, '127.0.0.1:' + localPort3) + const remoteProxy2 = tunnel.remote('127.0.0.1:' + remotePort4 + ':127.0.0.1:' + localPort4) await remoteProxy1.ready() await remoteProxy2.ready() From 2cf872c51cb95899574931cfa41157c191287398 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Sun, 20 Oct 2024 03:33:56 -0300 Subject: [PATCH 17/21] Fix docs about tunnel format --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dc4f172..d608297 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ Create a local proxy where every connection is forwarded to the server. Example: Access a private service in the server but locally e.g. a database port. ```sh -hypershell tunnel home -L 3000:127.0.0.1:3306:127.0.0.1 +hypershell tunnel home -L 127.0.0.1:3000:127.0.0.1:3306 ``` #### Remote port forwarding @@ -170,7 +170,7 @@ Create a remote proxy where every connection is forwarded locally. Example: Expose your local development React.js app to the internet. ```sh -hypershell tunnel home -R 80:0.0.0.0:3000:127.0.0.1 +hypershell tunnel home -R 0.0.0.0:80:127.0.0.1:3000 ``` #### Multiple tunnels at once @@ -178,7 +178,7 @@ hypershell tunnel home -R 80:0.0.0.0:3000:127.0.0.1 You can do this with both `-L` and `-R`. ```sh -hypershell tunnel home -L 5000:5900:127.0.0.1 -L 3000:3389:127.0.0.1 +hypershell tunnel home -L 5000:127.0.0.1:5900 -L 3000:127.0.0.1:3389 ``` #### Restrict tunnel server From a6b2507af6e806dadf6ec1167c5de9d71769de16 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Mon, 21 Oct 2024 01:47:56 -0300 Subject: [PATCH 18/21] Keys management (keys list, keys add, keys allow) --- .gitignore | 3 +- bin.js | 1 + bin/keygen.js | 2 +- bin/keys.js | 21 +++++++ lib/bin/copy.js | 5 +- lib/bin/keygen.js | 23 ++++++-- lib/bin/keys-add.js | 16 ++++++ lib/bin/keys-allow.js | 37 ++++++++++++ lib/bin/keys-list.js | 32 +++++++++++ lib/bin/login.js | 5 +- lib/bin/server.js | 3 +- lib/bin/tunnel.js | 5 +- lib/get-known-peer.js | 44 -------------- lib/get-primary-keys.js | 32 +++++++++++ lib/known-peers.js | 123 ++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 16 files changed, 293 insertions(+), 60 deletions(-) create mode 100644 bin/keys.js create mode 100644 lib/bin/keys-add.js create mode 100644 lib/bin/keys-allow.js create mode 100644 lib/bin/keys-list.js delete mode 100644 lib/get-known-peer.js create mode 100644 lib/get-primary-keys.js create mode 100644 lib/known-peers.js diff --git a/.gitignore b/.gitignore index 9770d6e..d5f19d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -node_modules/ +node_modules package-lock.json -coverage/ diff --git a/bin.js b/bin.js index 3e64ac3..82bfe52 100755 --- a/bin.js +++ b/bin.js @@ -12,6 +12,7 @@ const main = program .addCommand(require('./bin/copy.js')) .addCommand(require('./bin/tunnel.js')) .addCommand(require('./bin/keygen.js')) + .addCommand(require('./bin/keys.js')) main.parseAsync().catch(err => { safetyCatch(err) diff --git a/bin/keygen.js b/bin/keygen.js index 70a7723..83b1ef0 100644 --- a/bin/keygen.js +++ b/bin/keygen.js @@ -3,5 +3,5 @@ const { createCommand } = require('commander') module.exports = createCommand('keygen') .description('create keys of type ed25519') .option('-f ', 'filename of the seed key file') - .option('-c ', 'provides a new comment') + .option('-c ', 'provides a comment') .action(require('../lib/bin/keygen.js')) diff --git a/bin/keys.js b/bin/keys.js new file mode 100644 index 0000000..45263a8 --- /dev/null +++ b/bin/keys.js @@ -0,0 +1,21 @@ +const { createCommand } = require('commander') + +const keys = createCommand('keys') + .description('keys management') + +keys + .command('list') + .description('list keys') + .action(require('../lib/bin/keys-list.js')) + +keys + .command('add ') + .description('add a known peer by name') + .action(require('../lib/bin/keys-add.js')) + +keys + .command('allow ') + .description('authorize a peer into the server') + .action(require('../lib/bin/keys-allow.js')) + +module.exports = keys diff --git a/lib/bin/copy.js b/lib/bin/copy.js index dbf861b..ac64ab8 100644 --- a/lib/bin/copy.js +++ b/lib/bin/copy.js @@ -2,7 +2,7 @@ const fs = require('fs') const path = require('path') const constants = require('../constants.js') const keygen = require('./keygen.js') -const getKnownPeer = require('../get-known-peer.js') +const KnownPeers = require('../known-peers.js') const fileToKeyPair = require('../file-to-keypair.js') const Hypershell = require('../../index.js') @@ -21,7 +21,8 @@ module.exports = async function copy (source, destination, opts = {}) { source = parseRemotePath(source)[1] destination = parseRemotePath(destination)[1] - const serverPublicKey = await getKnownPeer(keyOrName, { verbose: true }) + const knownPeers = new KnownPeers({ cwd: constants.dir }) + const serverPublicKey = await knownPeers.getPublicKey(keyOrName) const hs = new Hypershell({ bootstrap: opts.bootstrap diff --git a/lib/bin/keygen.js b/lib/bin/keygen.js index a05eec5..56911f9 100644 --- a/lib/bin/keygen.js +++ b/lib/bin/keygen.js @@ -2,6 +2,7 @@ const fs = require('fs') const path = require('path') const crypto = require('hypercore-crypto') const HypercoreId = require('hypercore-id-encoding') +const crayon = require('tiny-crayon') const constants = require('../constants.js') const question = require('../question.js') @@ -11,20 +12,30 @@ module.exports = async function keygen (opts = {}) { comment = opts.comment ? (' # ' + opts.comment) : '' } = opts - console.log('Generating key.', { filename }) - if (!opts.f) { - const answer = await question('Enter file in which to save the key (' + filename + '): ') + console.log('Enter file, "id_" will be prefixed unless the path is absolute or starts with a dot') + + const answer = await question('(' + filename + '): ') if (answer) { filename = answer } } + const isName = !path.isAbsolute(filename) && filename[0] !== '.' + + if (isName) { + if (!filename.startsWith('id')) { + filename = 'id_' + filename + } + + filename = path.join(constants.dir, filename) + } + filename = path.resolve(filename) if (fs.existsSync(filename)) { - throw new Error('File already exists:' + filename) + throw new Error('File already exists: ' + filename) } const seed = crypto.randomBytes(32) @@ -33,9 +44,9 @@ module.exports = async function keygen (opts = {}) { await fs.promises.mkdir(path.dirname(filename), { recursive: true }) await fs.promises.writeFile(filename, HypercoreId.encode(seed) + comment + '\n', { flag: 'wx', mode: '600' }) - console.log('Your key has been saved in', filename) + console.log('Your key has been saved in', crayon.green(filename)) console.log('The public key is:') - console.log(HypercoreId.encode(keyPair.publicKey)) + console.log(crayon.green(HypercoreId.encode(keyPair.publicKey))) return keyPair } diff --git a/lib/bin/keys-add.js b/lib/bin/keys-add.js new file mode 100644 index 0000000..fc4e0c8 --- /dev/null +++ b/lib/bin/keys-add.js @@ -0,0 +1,16 @@ +const crayon = require('tiny-crayon') +const HypercoreId = require('hypercore-id-encoding') +const KnownPeers = require('../known-peers.js') +const constants = require('../constants.js') + +module.exports = async function keysAdd (name, publicKey, opts = {}) { + publicKey = HypercoreId.normalize(publicKey) + + const knownPeers = new KnownPeers({ cwd: constants.dir }) + + await knownPeers.put(name, publicKey) + + if (!opts.silent) { + console.log('Peer added', crayon.magenta(name), crayon.green(publicKey)) + } +} diff --git a/lib/bin/keys-allow.js b/lib/bin/keys-allow.js new file mode 100644 index 0000000..4097ac1 --- /dev/null +++ b/lib/bin/keys-allow.js @@ -0,0 +1,37 @@ +const fs = require('fs') +const path = require('path') +const crayon = require('tiny-crayon') +const configs = require('tiny-configs') +const HypercoreId = require('hypercore-id-encoding') +const KnownPeers = require('../known-peers.js') +const constants = require('../constants.js') + +module.exports = async function keysAllow (keyOrName, opts = {}) { + const filename = path.resolve(opts.firewall || path.join(constants.dir, 'authorized_peers')) + + if (!fs.existsSync(filename)) { + console.log('Notice: Creating default firewall', crayon.green(filename)) + + await fs.promises.mkdir(path.dirname(filename), { recursive: true }) + await fs.promises.writeFile(filename, '# \n', { flag: 'wx' }) + } + + const knownPeers = new KnownPeers({ cwd: constants.dir }) + const publicKey = HypercoreId.encode(await knownPeers.getPublicKey(keyOrName)) + + const content = await fs.promises.readFile(filename, 'utf8') + const authorized = configs.parse(content).map(HypercoreId.normalize) + + for (const authorizedPublicKey of authorized) { + if (authorizedPublicKey === publicKey) { + throw new Error('Public key is already allowed in the firewall') + } + } + + const name = await knownPeers.getNameByPublicKey(publicKey) + const comment = name ? (' # ' + name) : '' + + await fs.promises.appendFile(filename, publicKey + comment + '\n', { flag: 'a' }) + + console.log('Peer allowed: ' + (name ? crayon.magenta(name) + ' ' : '') + crayon.green(publicKey)) +} diff --git a/lib/bin/keys-list.js b/lib/bin/keys-list.js new file mode 100644 index 0000000..83003e0 --- /dev/null +++ b/lib/bin/keys-list.js @@ -0,0 +1,32 @@ +const crayon = require('tiny-crayon') +const getPrimaryKeys = require('../get-primary-keys.js') +const KnownPeers = require('../known-peers.js') +const constants = require('../constants.js') + +module.exports = async function keysList (opts = {}) { + console.log('Public keys:') + + const primaryKeys = await getPrimaryKeys() + + if (primaryKeys.length) { + for (const primary of primaryKeys) { + console.log('-', crayon.magenta(primary.name), crayon.green(primary.publicKey)) + } + } else { + console.log('- No keys found') + } + + console.log() + console.log('Known peers:') + + const knownPeers = new KnownPeers({ cwd: constants.dir }) + const peers = await knownPeers.list() + + if (peers.length) { + for (const peer of peers) { + console.log('-', crayon.magenta(peer.name), crayon.green(peer.publicKey)) + } + } else { + console.log('- No keys found') + } +} diff --git a/lib/bin/login.js b/lib/bin/login.js index c643be2..9a82221 100644 --- a/lib/bin/login.js +++ b/lib/bin/login.js @@ -5,7 +5,7 @@ const z32 = require('z32') const constants = require('../constants.js') const keygen = require('./keygen.js') const Hypershell = require('../../index.js') -const getKnownPeer = require('../get-known-peer.js') +const KnownPeers = require('../known-peers.js') const fileToKeyPair = require('../file-to-keypair.js') module.exports = async function login (keyOrName, opts = {}) { @@ -15,7 +15,8 @@ module.exports = async function login (keyOrName, opts = {}) { await keygen({ f: keyFilename }) } - const serverPublicKey = await getKnownPeer(keyOrName, { verbose: true }) + const knownPeers = new KnownPeers({ cwd: constants.dir }) + const serverPublicKey = await knownPeers.getPublicKey(keyOrName) const hs = new Hypershell({ bootstrap: opts.bootstrap diff --git a/lib/bin/server.js b/lib/bin/server.js index 7773637..0dcaaf6 100644 --- a/lib/bin/server.js +++ b/lib/bin/server.js @@ -4,6 +4,7 @@ const goodbye = require('graceful-goodbye') const readFile = require('read-file-live') const HypercoreId = require('hypercore-id-encoding') const configs = require('tiny-configs') +const crayon = require('tiny-crayon') const constants = require('../constants.js') const keygen = require('./keygen.js') const Hypershell = require('../../index.js') @@ -19,7 +20,7 @@ module.exports = async function server (opts = {}) { } if (firewallEnabled && !fs.existsSync(firewallFilename)) { - console.log('Notice: creating default firewall', firewallFilename) + console.log('Notice: Creating default firewall', crayon.green(firewallFilename)) await fs.promises.mkdir(path.dirname(firewallFilename), { recursive: true }) await fs.promises.writeFile(firewallFilename, '# \n', { flag: 'wx' }) diff --git a/lib/bin/tunnel.js b/lib/bin/tunnel.js index 531ec28..79e4aed 100644 --- a/lib/bin/tunnel.js +++ b/lib/bin/tunnel.js @@ -3,7 +3,7 @@ const path = require('path') const goodbye = require('graceful-goodbye') const constants = require('../constants.js') const keygen = require('./keygen.js') -const getKnownPeer = require('../get-known-peer.js') +const KnownPeers = require('../known-peers.js') const fileToKeyPair = require('../file-to-keypair.js') const Hypershell = require('../../index.js') @@ -18,7 +18,8 @@ module.exports = async function tunnel (keyOrName, opts = {}) { throw new Error('-L o -R is required') } - const serverPublicKey = await getKnownPeer(keyOrName, { verbose: true }) + const knownPeers = new KnownPeers({ cwd: constants.dir }) + const serverPublicKey = await knownPeers.getPublicKey(keyOrName) const hs = new Hypershell({ bootstrap: opts.bootstrap }) diff --git a/lib/get-known-peer.js b/lib/get-known-peer.js deleted file mode 100644 index 43cd43e..0000000 --- a/lib/get-known-peer.js +++ /dev/null @@ -1,44 +0,0 @@ -const fs = require('fs') -const path = require('path') -const configs = require('tiny-configs') -const HypercoreId = require('hypercore-id-encoding') -const constants = require('./constants.js') - -module.exports = async function getKnownPeer (host, opts) { - const peers = await readKnownPeers(opts) - - for (const peer of peers) { - if (peer.name === host) { - host = peer.publicKey - break - } - } - - return HypercoreId.decode(host) -} - -async function readKnownPeers (opts) { - const filename = path.join(constants.dir, 'known_peers') - - if (!fs.existsSync(filename)) { - if (opts && opts.verbose) { - console.log('Notice: creating default known peers', filename) - } - - await fs.promises.mkdir(path.dirname(filename), { recursive: true }) - await fs.promises.writeFile(filename, '# \n', { flag: 'wx' }) - } - - try { - const file = await fs.promises.readFile(filename, 'utf8') - - return configs.parse(file, { split: ' ', length: 2 }) - .map(m => ({ name: m[0], publicKey: m[1] })) - } catch (error) { - if (error.code === 'ENOENT') { - return [] - } - - throw error - } -} diff --git a/lib/get-primary-keys.js b/lib/get-primary-keys.js new file mode 100644 index 0000000..214c062 --- /dev/null +++ b/lib/get-primary-keys.js @@ -0,0 +1,32 @@ +const fs = require('fs') +const path = require('path') +const HypercoreId = require('hypercore-id-encoding') +const fileToKeyPair = require('./file-to-keypair.js') +const constants = require('./constants.js') + +module.exports = async function getPrimaryKeys () { + const files = await readdir(constants.dir) + const primaryKeys = [] + + for (const dirent of files) { + if (!dirent.name.startsWith('id')) { + continue + } + + const filename = path.join(constants.dir, dirent.name) + const keyPair = await fileToKeyPair(filename) + const publicKey = HypercoreId.encode(keyPair.publicKey) + + primaryKeys.push({ name: dirent.name, publicKey }) + } + + return primaryKeys +} + +async function readdir (dir) { + try { + return await fs.promises.readdir(dir, { withFileTypes: true }) + } catch { + return [] + } +} diff --git a/lib/known-peers.js b/lib/known-peers.js new file mode 100644 index 0000000..b9b49f8 --- /dev/null +++ b/lib/known-peers.js @@ -0,0 +1,123 @@ +const fs = require('fs') +const path = require('path') +const configs = require('tiny-configs') +const HypercoreId = require('hypercore-id-encoding') +const ReadyResource = require('ready-resource') +const safetyCatch = require('safety-catch') +const getPrimaryKeys = require('./get-primary-keys.js') + +module.exports = class KnownPeers extends ReadyResource { + constructor (opts = {}) { + super() + + this.filename = path.join(opts.cwd, 'known_peers') + + this.ready().catch(safetyCatch) + } + + async _open () { + if (fs.existsSync(this.filename)) { + return + } + + await fs.promises.mkdir(path.dirname(this.filename), { recursive: true }) + await fs.promises.writeFile(this.filename, '# \n', { flag: 'wx' }) + } + + async list () { + if (this.opened === false) await this.opening + + let file = null + + try { + file = await fs.promises.readFile(this.filename, 'utf8') + } catch (err) { + if (err.code === 'ENOENT') { + return [] + } + + throw err + } + + // TODO: Parse should also return the line index for del() + const parsed = configs.parse(file, { split: ' ', length: 2 }) + + return parsed.map(m => ({ name: m[0], publicKey: m[1] })) + } + + async get (name) { + name = name.trim() + + const peers = await this.list() + + for (const peer of peers) { + if (name === peer.name) { + return peer + } + } + + return null + } + + async put (name, publicKey) { + name = name.trim() + publicKey = HypercoreId.normalize(publicKey.trim()) + + if (name.startsWith('id')) { + throw new Error('Peer name can not start with "id"') + } + + const peers = await this.list() + + for (const peer of peers) { + if (name === peer.name) { + throw new Error('Peer name already exists: ' + peer.name) + } + + if (publicKey === peer.publicKey) { + throw new Error('Peer key already exists: (' + peer.name + ') ' + peer.publicKey) + } + } + + await fs.promises.appendFile(this.filename, name + ' ' + publicKey + '\n', { flag: 'a' }) + } + + async getPublicKey (host) { + const peer = await this.get(host) + + try { + // Match known peer by name + if (peer) { + return HypercoreId.decode(peer.publicKey) + } + + // Match primary key by filename like "id" or "id_server" + const primaryKeys = await getPrimaryKeys() + + for (const primary of primaryKeys) { + if (host === primary.name) { + return HypercoreId.decode(primary.publicKey) + } + } + + // Direct public key + return HypercoreId.decode(host) + } catch (err) { + safetyCatch(err) + + throw new Error('Peer name not found or invalid public key') + } + } + + async getNameByPublicKey (publicKey) { + const peers = await this.list() + + for (const peer of peers) { + if (publicKey === peer.publicKey) { + return peer.name + } + } + + return null + } +} diff --git a/package.json b/package.json index 9a694c7..6452452 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "ready-resource": "^1.1.1", "tar-fs": "^3.0.6", "tiny-configs": "^1.1.0", + "tiny-crayon": "^1.0.5", "tt-native": "^1.1.1", "z32": "^1.1.0" } From b28149512fff7260bf0fc46ee8bed2a1898de97e Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Mon, 21 Oct 2024 01:59:26 -0300 Subject: [PATCH 19/21] Colors --- index.js | 9 +++++---- lib/bin/login.js | 3 ++- lib/bin/server.js | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index d4c38fb..a7a6d7e 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ const Protomux = require('protomux') const HypercoreId = require('hypercore-id-encoding') const crypto = require('hypercore-crypto') const safetyCatch = require('safety-catch') +const crayon = require('tiny-crayon') const { ShellServer, ShellClient } = require('./lib/protocols/shell.js') const Copy = require('./lib/protocols/copy.js') const Tunnel = require('./lib/protocols/tunnel.js') @@ -137,7 +138,7 @@ class Server { this._connections.add(socket) if (this.verbose) { - console.log('Connection opened', HypercoreId.encode(socket.remotePublicKey)) + console.log(crayon.green('Connection opened'), HypercoreId.encode(socket.remotePublicKey), '(' + this._connections.size + ')') } socket.setKeepAlive(5000) @@ -148,7 +149,7 @@ class Server { this._connections.delete(socket) if (this.verbose) { - console.log('Connection closed', HypercoreId.encode(socket.remotePublicKey)) + console.log(crayon.gray('Connection closed'), HypercoreId.encode(socket.remotePublicKey), '(' + this._connections.size + ')') } }) @@ -167,7 +168,7 @@ class Server { for (const [publicKey] of this.invites) { if (remotePublicKey.equals(Buffer.from(publicKey, 'hex'))) { if (this.verbose) { - console.log('Invite accepted:', HypercoreId.encode(remotePublicKey)) + console.log(crayon.cyan('Invite accepted'), HypercoreId.encode(remotePublicKey)) } this.invites.delete(publicKey) @@ -183,7 +184,7 @@ class Server { } if (this.verbose) { - console.log('Firewall denied:', HypercoreId.encode(remotePublicKey)) + console.log(crayon.red('Connection denied'), HypercoreId.encode(remotePublicKey)) } return true diff --git a/lib/bin/login.js b/lib/bin/login.js index 9a82221..0c184b6 100644 --- a/lib/bin/login.js +++ b/lib/bin/login.js @@ -2,6 +2,7 @@ const fs = require('fs') const path = require('path') const crypto = require('hypercore-crypto') const z32 = require('z32') +const crayon = require('tiny-crayon') const constants = require('../constants.js') const keygen = require('./keygen.js') const Hypershell = require('../../index.js') @@ -29,7 +30,7 @@ module.exports = async function login (keyOrName, opts = {}) { const invite = await admin.createInvite() - console.log('One time invite:', z32.encode(invite)) + console.log('One time invite:', crayon.cyan(z32.encode(invite))) await admin.close() await hs.destroy() diff --git a/lib/bin/server.js b/lib/bin/server.js index 0dcaaf6..a03dc39 100644 --- a/lib/bin/server.js +++ b/lib/bin/server.js @@ -51,7 +51,7 @@ module.exports = async function server (opts = {}) { if (server.protocols.includes('shell')) { console.log('To connect to this shell, on another computer run:') - console.log('hypershell login ' + HypercoreId.encode(server.publicKey)) + console.log(crayon.cyan('hypershell login ' + HypercoreId.encode(server.publicKey))) } else { console.log('Running server with restricted protocols:', server.protocols.join(', ')) console.log('Server key: ' + HypercoreId.encode(server.publicKey)) From 0e08df32250bd71a5e88686eab6a8109c975d972 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Mon, 21 Oct 2024 02:07:52 -0300 Subject: [PATCH 20/21] Dir of known peers is always the same --- bin/keys.js | 1 + lib/bin/copy.js | 2 +- lib/bin/keys-add.js | 3 +-- lib/bin/keys-allow.js | 2 +- lib/bin/keys-list.js | 3 +-- lib/bin/login.js | 2 +- lib/bin/tunnel.js | 2 +- lib/known-peers.js | 5 +++-- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/keys.js b/bin/keys.js index 45263a8..92300f2 100644 --- a/bin/keys.js +++ b/bin/keys.js @@ -16,6 +16,7 @@ keys keys .command('allow ') .description('authorize a peer into the server') + .option('--firewall ', 'list of allowed public keys') .action(require('../lib/bin/keys-allow.js')) module.exports = keys diff --git a/lib/bin/copy.js b/lib/bin/copy.js index ac64ab8..c25f55c 100644 --- a/lib/bin/copy.js +++ b/lib/bin/copy.js @@ -21,7 +21,7 @@ module.exports = async function copy (source, destination, opts = {}) { source = parseRemotePath(source)[1] destination = parseRemotePath(destination)[1] - const knownPeers = new KnownPeers({ cwd: constants.dir }) + const knownPeers = new KnownPeers() const serverPublicKey = await knownPeers.getPublicKey(keyOrName) const hs = new Hypershell({ diff --git a/lib/bin/keys-add.js b/lib/bin/keys-add.js index fc4e0c8..f85cfc3 100644 --- a/lib/bin/keys-add.js +++ b/lib/bin/keys-add.js @@ -1,12 +1,11 @@ const crayon = require('tiny-crayon') const HypercoreId = require('hypercore-id-encoding') const KnownPeers = require('../known-peers.js') -const constants = require('../constants.js') module.exports = async function keysAdd (name, publicKey, opts = {}) { publicKey = HypercoreId.normalize(publicKey) - const knownPeers = new KnownPeers({ cwd: constants.dir }) + const knownPeers = new KnownPeers() await knownPeers.put(name, publicKey) diff --git a/lib/bin/keys-allow.js b/lib/bin/keys-allow.js index 4097ac1..057fbf4 100644 --- a/lib/bin/keys-allow.js +++ b/lib/bin/keys-allow.js @@ -16,7 +16,7 @@ module.exports = async function keysAllow (keyOrName, opts = {}) { await fs.promises.writeFile(filename, '# \n', { flag: 'wx' }) } - const knownPeers = new KnownPeers({ cwd: constants.dir }) + const knownPeers = new KnownPeers() const publicKey = HypercoreId.encode(await knownPeers.getPublicKey(keyOrName)) const content = await fs.promises.readFile(filename, 'utf8') diff --git a/lib/bin/keys-list.js b/lib/bin/keys-list.js index 83003e0..a4ddae9 100644 --- a/lib/bin/keys-list.js +++ b/lib/bin/keys-list.js @@ -1,7 +1,6 @@ const crayon = require('tiny-crayon') const getPrimaryKeys = require('../get-primary-keys.js') const KnownPeers = require('../known-peers.js') -const constants = require('../constants.js') module.exports = async function keysList (opts = {}) { console.log('Public keys:') @@ -19,7 +18,7 @@ module.exports = async function keysList (opts = {}) { console.log() console.log('Known peers:') - const knownPeers = new KnownPeers({ cwd: constants.dir }) + const knownPeers = new KnownPeers() const peers = await knownPeers.list() if (peers.length) { diff --git a/lib/bin/login.js b/lib/bin/login.js index 0c184b6..08739af 100644 --- a/lib/bin/login.js +++ b/lib/bin/login.js @@ -16,7 +16,7 @@ module.exports = async function login (keyOrName, opts = {}) { await keygen({ f: keyFilename }) } - const knownPeers = new KnownPeers({ cwd: constants.dir }) + const knownPeers = new KnownPeers() const serverPublicKey = await knownPeers.getPublicKey(keyOrName) const hs = new Hypershell({ diff --git a/lib/bin/tunnel.js b/lib/bin/tunnel.js index 79e4aed..c04bd22 100644 --- a/lib/bin/tunnel.js +++ b/lib/bin/tunnel.js @@ -18,7 +18,7 @@ module.exports = async function tunnel (keyOrName, opts = {}) { throw new Error('-L o -R is required') } - const knownPeers = new KnownPeers({ cwd: constants.dir }) + const knownPeers = new KnownPeers() const serverPublicKey = await knownPeers.getPublicKey(keyOrName) const hs = new Hypershell({ bootstrap: opts.bootstrap }) diff --git a/lib/known-peers.js b/lib/known-peers.js index b9b49f8..c153902 100644 --- a/lib/known-peers.js +++ b/lib/known-peers.js @@ -4,13 +4,14 @@ const configs = require('tiny-configs') const HypercoreId = require('hypercore-id-encoding') const ReadyResource = require('ready-resource') const safetyCatch = require('safety-catch') +const constants = require('./constants.js') const getPrimaryKeys = require('./get-primary-keys.js') module.exports = class KnownPeers extends ReadyResource { - constructor (opts = {}) { + constructor () { super() - this.filename = path.join(opts.cwd, 'known_peers') + this.filename = path.join(constants.dir, 'known_peers') this.ready().catch(safetyCatch) } From fca7c65d670d104cfc778be10f80888dcae846e8 Mon Sep 17 00:00:00 2001 From: Lucas Barrena Date: Mon, 21 Oct 2024 02:09:49 -0300 Subject: [PATCH 21/21] Remove author --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6452452..9f9c312 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "type": "git", "url": "git+https://github.com/holepunchto/hypershell.git" }, - "author": "Lucas Barrena (@LuKks)", + "author": "Holepunch", "license": "Apache-2.0", "bugs": { "url": "https://github.com/holepunchto/hypershell/issues"