From c310af822def12b365d1dd7abb0ad556d610689c Mon Sep 17 00:00:00 2001 From: Mathias Buus Date: Wed, 6 Feb 2019 12:59:11 +0100 Subject: [PATCH 001/108] Initial commit --- .gitignore | 64 ++- README.md | 326 +------------- index.js | 950 ----------------------------------------- lib/cursor.js | 142 ------ lib/messages.js | 278 ------------ lib/stat.js | 48 --- package.json | 54 +-- schema.proto | 16 - test/basic.js | 193 --------- test/cursor.js | 87 ---- test/helpers/create.js | 6 - test/stat.js | 37 -- test/storage.js | 134 ------ 13 files changed, 90 insertions(+), 2245 deletions(-) delete mode 100644 index.js delete mode 100644 lib/cursor.js delete mode 100644 lib/messages.js delete mode 100644 lib/stat.js delete mode 100644 schema.proto delete mode 100644 test/basic.js delete mode 100644 test/cursor.js delete mode 100644 test/helpers/create.js delete mode 100644 test/stat.js delete mode 100644 test/storage.js diff --git a/.gitignore b/.gitignore index 0dd17314..ad46b308 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,61 @@ -node_modules -sandbox.js -sandbox +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + +# next.js build output +.next diff --git a/README.md b/README.md index 935b814e..ab33cd61 100644 --- a/README.md +++ b/README.md @@ -1,324 +1,2 @@ -# Hyperdrive - -Hyperdrive is a secure, real time distributed file system - -``` js -npm install hyperdrive -``` - -[![Build Status](https://travis-ci.org/mafintosh/hyperdrive.svg?branch=master)](https://travis-ci.org/mafintosh/hyperdrive) - -## Usage - -Hyperdrive aims to implement the same API as Node.js' core fs module. - -``` js -var hyperdrive = require('hyperdrive') -var archive = hyperdrive('./my-first-hyperdrive') // content will be stored in this folder - -archive.writeFile('/hello.txt', 'world', function (err) { - if (err) throw err - archive.readdir('/', function (err, list) { - if (err) throw err - console.log(list) // prints ['hello.txt'] - archive.readFile('/hello.txt', 'utf-8', function (err, data) { - if (err) throw err - console.log(data) // prints 'world' - }) - }) -}) -``` - -A big difference is that you can replicate the file system to other computers! All you need is a stream. - -``` js -var net = require('net') - -// ... on one machine - -var server = net.createServer(function (socket) { - socket.pipe(archive.replicate()).pipe(socket) -}) - -server.listen(10000) - -// ... on another - -var clonedArchive = hyperdrive('./my-cloned-hyperdrive', origKey) -var socket = net.connect(10000) - -socket.pipe(clonedArchive.replicate()).pipe(socket) -``` - -It also comes with build in versioning and real time replication. See more below. - -## API - -#### `var archive = hyperdrive(storage, [key], [options])` - -Create a new hyperdrive. - -The `storage` parameter defines how the contents of the archive will be stored. It can be one of the following, depending on how much control you require over how the archive is stored. - -- If you pass in a string, the archive content will be stored in a folder at the given path. -- You can also pass in a function. This function will be called with the name of each of the required files for the archive, and needs to return a [`random-access-storage`](https://github.com/random-access-storage/) instance. -- If you require complete control, you can also pass in an object containing a `metadata` and a `content` field. Both of these need to be functions, and are called with the following arguments: - - - `name`: the name of the file to be stored - - `opts` - - `key`: the [feed key](https://github.com/mafintosh/hypercore#feedkey) of the underlying Hypercore instance - - `discoveryKey`: the [discovery key](https://github.com/mafintosh/hypercore#feeddiscoverykey) of the underlying Hypercore instance - - `archive`: the current Hyperdrive instance - - The functions need to return a a [`random-access-storage`](https://github.com/random-access-storage/) instance. - -Options include: - -``` js -{ - sparse: true, // only download data on content feed when it is specifically requested - sparseMetadata: true // only download data on metadata feed when requested - metadataStorageCacheSize: 65536 // how many entries to use in the metadata hypercore's LRU cache - contentStorageCacheSize: 65536 // how many entries to use in the content hypercore's LRU cache - treeCacheSize: 65536 // how many entries to use in the append-tree's LRU cache -} -``` - -Note that a cloned hyperdrive archive can be "sparse". Usually (by setting `sparse: true`) this means that the content is not downloaded until you ask for it, but the entire metadata feed is still downloaded. If you want a _very_ sparse archive, where even the metadata feed is not downloaded until you request it, then you should _also_ set `sparseMetadata: true`. - -#### `var stream = archive.replicate([options])` - -Replicate this archive. Options include - -``` js -{ - live: false, // keep replicating - download: true, // download data from peers? - upload: true // upload data to peers? -} -``` - -#### `archive.version` - -Get the current version of the archive (incrementing number). - -#### `archive.key` - -The public key identifying the archive. - -#### `archive.discoveryKey` - -A key derived from the public key that can be used to discovery other peers sharing this archive. - -#### `archive.writable` - -A boolean indicating whether the archive is writable. - -#### `archive.on('ready')` - -Emitted when the archive is fully ready and all properties has been populated. - -#### `archive.on('error', err)` - -Emitted when a critical error during load happened. - -#### `var oldDrive = archive.checkout(version, [opts])` - -Checkout a readonly copy of the archive at an old version. Options are used to configure the `oldDrive`: - -```js -{ - metadataStorageCacheSize: 65536 // how many entries to use in the metadata hypercore's LRU cache - contentStorageCacheSize: 65536 // how many entries to use in the content hypercore's LRU cache - treeCacheSize: 65536 // how many entries to use in the append-tree's LRU cache -} -``` - -#### `archive.download([path], [callback])` - -Download all files in path of current version. -If no path is specified this will download all files. - -You can use this with `.checkout(version)` to download a specific version of the archive. - -``` js -archive.checkout(version).download() -``` - -#### `var stream = archive.history([options])` - -Get a stream of all changes and their versions from this archive. - -#### `var stream = archive.createReadStream(name, [options])` - -Read a file out as a stream. Similar to fs.createReadStream. - -Options include: - -``` js -{ - start: optionalByteOffset, // similar to fs - end: optionalInclusiveByteEndOffset, // similar to fs - length: optionalByteLength -} -``` - -#### `archive.readFile(name, [options], callback)` - -Read an entire file into memory. Similar to fs.readFile. - -Options can either be an object or a string - -Options include: -```js -{ - encoding: string - cached: true|false // default: false -} -``` -or a string can be passed as options to simply set the encoding - similar to fs. - -If `cached` is set to `true`, this function returns results only if they have already been downloaded. - -#### `var stream = archive.createDiffStream(version, [options])` - -Diff this archive this another version. `version` can both be a version number of a checkout instance of the archive. The `data` objects looks like this - -``` js -{ - type: 'put' | 'del', - name: '/some/path/name.txt', - value: { - // the stat object - } -} -``` - -#### `var stream = archive.createWriteStream(name, [options])` - -Write a file as a stream. Similar to fs.createWriteStream. -If `options.cached` is set to `true`, this function returns results only if they have already been downloaded. - -#### `archive.writeFile(name, buffer, [options], [callback])` - -Write a file from a single buffer. Similar to fs.writeFile. - -#### `archive.unlink(name, [callback])` - -Unlinks (deletes) a file. Similar to fs.unlink. - -#### `archive.mkdir(name, [options], [callback])` - -Explictly create an directory. Similar to fs.mkdir - -#### `archive.rmdir(name, [callback])` - -Delete an empty directory. Similar to fs.rmdir. - -#### `archive.readdir(name, [options], [callback])` - -Lists a directory. Similar to fs.readdir. - -Options include: - -``` js -{ - cached: true|false, // default: false -} -``` - -If `cached` is set to `true`, this function returns results from the local version of the archive’s append-tree. Default behavior is to fetch the latest remote version of the archive before returning list of directories. - -#### `archive.stat(name, [options], callback)` - -Stat an entry. Similar to fs.stat. Sample output: - -``` -Stat { - dev: 0, - nlink: 1, - rdev: 0, - blksize: 0, - ino: 0, - mode: 16877, - uid: 0, - gid: 0, - size: 0, - offset: 0, - blocks: 0, - atime: 2017-04-10T18:59:00.147Z, - mtime: 2017-04-10T18:59:00.147Z, - ctime: 2017-04-10T18:59:00.147Z, - linkname: undefined } -``` - -The output object includes methods similar to fs.stat: - -``` js -var stat = archive.stat('/hello.txt') -stat.isDirectory() -stat.isFile() -``` - -Options include: -```js -{ - cached: true|false // default: false, - wait: true|false // default: true -} -``` - -If `cached` is set to `true`, this function returns results only if they have already been downloaded. - -If `wait` is set to `true`, this function will wait for data to be downloaded. If false, will return an error. - -#### `archive.lstat(name, [options], callback)` - -Stat an entry but do not follow symlinks. Similar to fs.lstat. - -Options include: -```js -{ - cached: true|false // default: false, - wait: true|false // default: true -} -``` - -If `cached` is set to `true`, this function returns results only if they have already been downloaded. - -If `wait` is set to `true`, this function will wait for data to be downloaded. If false, will return an error. - -#### `archive.access(name, [options], callback)` - -Similar to fs.access. - -Options include: -```js -{ - cached: true|false // default: false, - wait: true|false // default: true -} -``` - -If `cached` is set to `true`, this function returns results only if they have already been downloaded. - -If `wait` is set to `true`, this function will wait for data to be downloaded. If false, will return an error. - -#### `archive.open(name, flags, [mode], callback)` - -Open a file and get a file descriptor back. Similar to fs.open. - -Note that currently only read mode is supported in this API. - -#### `archive.read(fd, buf, offset, len, position, callback)` - -Read from a file descriptor into a buffer. Similar to fs.read. - -#### `archive.close(fd, [callback])` - -Close a file. Similar to fs.close. - -#### `archive.close([callback])` - -Closes all open resources used by the archive. -The archive should no longer be used after calling this. +# hypertrie-drive +A version of hyperdrive backed by hypertrie diff --git a/index.js b/index.js deleted file mode 100644 index afbeb7ba..00000000 --- a/index.js +++ /dev/null @@ -1,950 +0,0 @@ -var hypercore = require('hypercore') -var mutexify = require('mutexify') -var raf = require('random-access-file') -var thunky = require('thunky') -var tree = require('append-tree') -var collect = require('stream-collector') -var sodium = require('sodium-universal') -var inherits = require('inherits') -var events = require('events') -var duplexify = require('duplexify') -var from = require('from2') -var each = require('stream-each') -var uint64be = require('uint64be') -var unixify = require('unixify') -var path = require('path') -var messages = require('./lib/messages') -var stat = require('./lib/stat') -var cursor = require('./lib/cursor') - -var DEFAULT_FMODE = (4 | 2 | 0) << 6 | ((4 | 0 | 0) << 3) | (4 | 0 | 0) // rw-r--r-- -var DEFAULT_DMODE = (4 | 2 | 1) << 6 | ((4 | 0 | 1) << 3) | (4 | 0 | 1) // rwxr-xr-x - -module.exports = Hyperdrive - -function Hyperdrive (storage, key, opts) { - if (!(this instanceof Hyperdrive)) return new Hyperdrive(storage, key, opts) - events.EventEmitter.call(this) - - if (isObject(key)) { - opts = key - key = null - } - - if (!opts) opts = {} - - this.key = null - this.discoveryKey = null - this.live = true - this.latest = !!opts.latest - - this._storages = defaultStorage(this, storage, opts) - - this.metadata = opts.metadata || hypercore(this._storages.metadata, key, { - secretKey: opts.secretKey, - sparse: opts.sparseMetadata, - createIfMissing: opts.createIfMissing, - storageCacheSize: opts.metadataStorageCacheSize - }) - this.content = opts.content || null - this.maxRequests = opts.maxRequests || 16 - this.readable = true - - this.storage = storage - this.tree = tree(this.metadata, { - offset: 1, - valueEncoding: messages.Stat, - cache: opts.treeCacheSize !== 0, - cacheSize: opts.treeCacheSize - }) - if (typeof opts.version === 'number') this.tree = this.tree.checkout(opts.version) - this.sparse = !!opts.sparse - this.sparseMetadata = !!opts.sparseMetadata - this.indexing = !!opts.indexing - this.contentStorageCacheSize = opts.contentStorageCacheSize - - this._latestSynced = 0 - this._latestVersion = 0 - this._latestStorage = this.latest ? this._storages.metadata('latest') : null - this._checkout = opts._checkout - this._lock = mutexify() - - this._openFiles = [] - this._emittedContent = false - this._closed = false - - var self = this - - this.metadata.on('append', update) - this.metadata.on('error', onerror) - this.ready = thunky(open) - this.ready(onready) - - function onready (err) { - if (err) return onerror(err) - self.emit('ready') - self._oncontent() - if (self.latest && !self.metadata.writable) { - self._trackLatest(function (err) { - if (self._closed) return - onerror(err) - }) - } - } - - function onerror (err) { - if (err) self.emit('error', err) - } - - function update () { - self.emit('update') - } - - function open (cb) { - self._open(cb) - } -} - -inherits(Hyperdrive, events.EventEmitter) - -Object.defineProperty(Hyperdrive.prototype, 'version', { - enumerable: true, - get: function () { - return this._checkout ? this.tree.version : (this.metadata.length ? this.metadata.length - 1 : 0) - } -}) - -Object.defineProperty(Hyperdrive.prototype, 'writable', { - enumerable: true, - get: function () { - return this.metadata.writable - } -}) - -Hyperdrive.prototype._oncontent = function () { - if (!this.content || this._emittedContent) return - this._emittedContent = true - this.emit('content') -} - -Hyperdrive.prototype._trackLatest = function (cb) { - var self = this - - this.ready(function (err) { - if (err) return cb(err) - - self._latestStorage.read(0, 8, function (_, data) { - self._latestVersion = data ? uint64be.decode(data) : 0 - loop() - }) - }) - - function loop (err) { - if (err) return cb(err) - - if (stableVersion()) return fetch() - - // TODO: lock downloading while doing this - self._clearDangling(self._latestVersion, self.version, onclear) - } - - function fetch () { - if (self.sparse) { - if (stableVersion()) return self.metadata.update(loop) - return loop(null) - } - - self.emit('syncing') - self._fetchVersion(self._latestSynced, function (err, fullySynced) { - if (err) return cb(err) - - if (fullySynced) { - self._latestSynced = self._latestVersion - self.emit('sync') - if (!self._checkout) self.metadata.update(loop) // TODO: only if live - return - } - - loop(null) - }) - } - - function onclear (err, version) { - if (err) return cb(err) - self._latestVersion = version - self._latestStorage.write(0, uint64be.encode(self._latestVersion), loop) - } - - function stableVersion () { - var latest = self.version - return latest < 0 || self._latestVersion === latest - } -} - -Hyperdrive.prototype._fetchVersion = function (prev, cb) { - var self = this - var version = self.version - var updated = false - var done = false - var error = null - var stream = null - var queued = 0 - var maxQueued = 64 - - var waitingData = null - var waitingCallback = null - - this.metadata.update(function () { - updated = true - queued = 0 - if (stream) stream.destroy() - kick() - }) - - this._ensureContent(function (err) { - if (err) return cb(err) - if (updated) return cb(null, false) - - // var snapshot = self.checkout(version) - stream = self.tree.checkout(prev).diff(version, {puts: true, dels: false}) - each(stream, ondata, ondone) - }) - - function ondata (data, next) { - if (updated || error) return callAndKick(next, new Error('Out of date')) - - if (queued >= maxQueued) { - waitingData = data - waitingCallback = next - return - } - - var start = data.value.offset - var end = start + data.value.blocks - - if (start === end) return callAndKick(next, null) - - queued++ - self.content.download({start: start, end: end}, function (err) { - if (updated && !waitingCallback) return kick() - if (!updated) queued-- - - if (waitingCallback) { - data = waitingData - waitingData = null - next = waitingCallback - waitingCallback = null - return ondata(data, next) - } - - if (err) { - stream.destroy(err) - error = err - } - - kick() - }) - - process.nextTick(next) - } - - function callAndKick (next, err) { - next(err) - kick() - } - - function kick () { - if (!done || queued) return - queued = -1 // so we don't enter this twice - - if (updated) return cb(null, false) - if (error) return cb(error) - - cb(null, version === self.version) - } - - function ondone (err) { - if (err) error = err - done = true - kick() - } -} - -Hyperdrive.prototype._clearDangling = function (a, b, cb) { - var current = this.tree.checkout(a, {cached: true}) - var latest = this.tree.checkout(b) - var stream = current.diff(latest, {dels: true, puts: false}) - var self = this - - this._ensureContent(oncontent) - - function done (err) { - if (err) return cb(err) - cb(null, b) - } - - function oncontent (err) { - if (err) return cb(err) - each(stream, ondata, done) - } - - function ondata (data, next) { - var st = data.value - self.content.cancel(st.offset, st.offset + st.blocks) - self.content.clear(st.offset, st.offset + st.blocks, {byteOffset: st.byteOffset, byteLength: st.size}, next) - } -} - -Hyperdrive.prototype.replicate = function (opts) { - if (!opts) opts = {} - - opts.expectedFeeds = 2 - - var self = this - var stream = this.metadata.replicate(opts) - - this._ensureContent(function (err) { - if (err) return stream.destroy(err) - if (stream.destroyed) return - self.content.replicate({ - live: opts.live, - download: opts.download, - upload: opts.upload, - stream: stream - }) - }) - - return stream -} - -Hyperdrive.prototype.checkout = function (version, opts) { - if (!opts) opts = {} - opts._checkout = this._checkout || this - opts.metadata = this.metadata - opts.version = version - return Hyperdrive(null, null, opts) -} - -Hyperdrive.prototype.createDiffStream = function (version, opts) { - if (!version) version = 0 - if (typeof version === 'number') version = this.checkout(version) - return this.tree.diff(version.tree, opts) -} - -Hyperdrive.prototype.download = function (dir, cb) { - if (typeof dir === 'function') return this.download('/', dir) - - var downloadCount = 1 - var self = this - - download(dir || '/') - - function download (entry) { - self.stat(entry, function (err, stat) { - if (err) { - if (cb) cb(err) - return - } - if (stat.isDirectory()) return downloadDir(entry, stat) - if (stat.isFile()) return downloadFile(entry, stat) - }) - } - - function downloadDir (dirname, stat) { - self.readdir(dirname, function (err, entries) { - if (err) { - if (cb) cb(err) - return - } - downloadCount -= 1 - downloadCount += entries.length - entries.forEach(function (entry) { - download(path.join(dirname, entry)) - }) - if (downloadCount <= 0 && cb) cb() - }) - } - - function downloadFile (entry, stat) { - var start = stat.offset - var end = stat.offset + stat.blocks - if (start === 0 && end === 0) return - self.content.download({start, end}, function () { - downloadCount -= 1 - if (downloadCount <= 0 && cb) cb() - }) - } -} - -Hyperdrive.prototype.history = function (opts) { - return this.tree.history(opts) -} - -Hyperdrive.prototype.createCursor = function (name, opts) { - return cursor(this, name, opts) -} - -// open -> fd -Hyperdrive.prototype.open = function (name, flags, mode, opts, cb) { - if (typeof mode === 'object' && mode) return this.open(name, flags, 0, mode, opts) - if (typeof mode === 'function') return this.open(name, flags, 0, mode) - if (typeof opts === 'function') return this.open(name, flags, mode, null, opts) - - // TODO: use flags, only readable cursors are supported atm - var cursor = this.createCursor(name, opts) - var self = this - - cursor.open(function (err) { - if (err) return cb(err) - - var fd = self._openFiles.indexOf(null) - if (fd === -1) fd = self._openFiles.push(null) - 1 - - self._openFiles[fd] = cursor - cb(null, fd + 20) // offset all fds with 20, unsure what the actual good offset is - }) -} - -Hyperdrive.prototype.read = function (fd, buf, offset, len, pos, cb) { - var cursor = this._openFiles[fd - 20] - if (!cursor) return cb(new Error('Bad file descriptor')) - - if (pos !== null) cursor.seek(pos) - - cursor.next(function (err, next) { - if (err) return cb(err) - - if (!next) return cb(null, 0, buf) - - // if we read too much - if (next.length > len) { - next = next.slice(0, len) - cursor.seek(pos + len) - } - - next.copy(buf, offset, 0, len) - cb(null, next.length, buf) - }) -} - -// TODO: move to ./lib -Hyperdrive.prototype.createReadStream = function (name, opts) { - if (!opts) opts = {} - - name = unixify(name) - - var self = this - var downloaded = false - var first = true - var start = 0 - var end = 0 - var offset = 0 - var length = typeof opts.end === 'number' ? 1 + opts.end - (opts.start || 0) : typeof opts.length === 'number' ? opts.length : -1 - var range = null - var ended = false - var stream = from(read) - var cached = opts && !!opts.cached - - stream.on('close', cleanup) - stream.on('end', cleanup) - - return stream - - function cleanup () { - if (range) self.content.undownload(range, noop) - range = null - ended = true - } - - function read (size, cb) { - if (first) return open(size, cb) - if (start === end || length === 0) return cb(null, null) - - self.content.get(start++, {wait: !downloaded && !cached}, function (err, data) { - if (err) return cb(err) - if (offset) data = data.slice(offset) - offset = 0 - if (length > -1) { - if (length < data.length) data = data.slice(0, length) - length -= data.length - } - cb(null, data) - }) - } - - function open (size, cb) { - first = false - self._ensureContent(function (err) { - if (err) return cb(err) - - // if running latest === true and a delete happens while getting the tree data, the tree.get - // should finish before the delete so there shouldn't be an rc. we should test this though. - self.tree.get(name, ontree) - - function ontree (err, stat) { - if (err) return cb(err) - if (ended || stream.destroyed) return - - start = stat.offset - end = stat.offset + stat.blocks - - var byteOffset = stat.byteOffset - var missing = 1 - - if (opts.start) self.content.seek(byteOffset + opts.start, {start: start, end: end}, onstart) - else onstart(null, start, 0) - - function onend (err, index) { - if (err || !range) return - if (ended || stream.destroyed) return - - missing++ - self.content.undownload(range) - range = self.content.download({start: start, end: index, linear: true}, ondownload) - } - - function onstart (err, index, off) { - if (err) return cb(err) - if (ended || stream.destroyed) return - - offset = off - start = index - range = self.content.download({start: start, end: end, linear: true}, ondownload) - - if (length > -1 && length < stat.size) { - self.content.seek(byteOffset + length, {start: start, end: end}, onend) - } - - read(size, cb) - } - - function ondownload (err) { - if (--missing) return - if (err && !ended && !downloaded) stream.destroy(err) - else downloaded = true - } - } - }) - } -} - -Hyperdrive.prototype.readFile = function (name, opts, cb) { - if (typeof opts === 'function') return this.readFile(name, null, opts) - if (typeof opts === 'string') opts = {encoding: opts} - if (!opts) opts = {} - - name = unixify(name) - - collect(this.createReadStream(name, opts), function (err, bufs) { - if (err) return cb(err) - var buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs) - cb(null, opts.encoding && opts.encoding !== 'binary' ? buf.toString(opts.encoding) : buf) - }) -} - -Hyperdrive.prototype.createWriteStream = function (name, opts) { - if (!opts) opts = {} - - name = unixify(name) - - var self = this - var proxy = duplexify() - - // TODO: support piping through a "split" stream like rabin - - proxy.setReadable(false) - this._ensureContent(function (err) { - if (err) return proxy.destroy(err) - if (self._checkout) return proxy.destroy(new Error('Cannot write to a checkout')) - if (proxy.destroyed) return - - self._lock(function (release) { - if (!self.latest || proxy.destroyed) return append(null) - - self.tree.get(name, function (err, st) { - if (err && err.notFound) return append(null) - if (err) return append(err) - if (!st.size) return append(null) - self.content.clear(st.offset, st.offset + st.blocks, append) - }) - - function append (err) { - if (err) proxy.destroy(err) - if (proxy.destroyed) return release() - - // No one should mutate the content other than us - var byteOffset = self.content.byteLength - var offset = self.content.length - - self.emit('appending', name, opts) - - // TODO: revert the content feed if this fails!!!! (add an option to the write stream for this (atomic: true)) - var stream = self.content.createWriteStream() - - proxy.on('close', done) - proxy.on('finish', done) - - proxy.setWritable(stream) - proxy.on('prefinish', function () { - var st = { - mode: (opts.mode || DEFAULT_FMODE) | stat.IFREG, - uid: opts.uid || 0, - gid: opts.gid || 0, - size: self.content.byteLength - byteOffset, - blocks: self.content.length - offset, - offset: offset, - byteOffset: byteOffset, - mtime: getTime(opts.mtime), - ctime: getTime(opts.ctime) - } - - proxy.cork() - self.tree.put(name, st, function (err) { - if (err) return proxy.destroy(err) - self.emit('append', name, opts) - proxy.uncork() - }) - }) - } - - function done () { - proxy.removeListener('close', done) - proxy.removeListener('finish', done) - release() - } - }) - }) - - return proxy -} - -Hyperdrive.prototype.writeFile = function (name, buf, opts, cb) { - if (typeof opts === 'function') return this.writeFile(name, buf, null, opts) - if (typeof opts === 'string') opts = {encoding: opts} - if (!opts) opts = {} - if (typeof buf === 'string') buf = new Buffer(buf, opts.encoding || 'utf-8') - if (!cb) cb = noop - - name = unixify(name) - - var bufs = split(buf) // split the input incase it is a big buffer. - var stream = this.createWriteStream(name, opts) - stream.on('error', cb) - stream.on('finish', cb) - for (var i = 0; i < bufs.length; i++) stream.write(bufs[i]) - stream.end() -} - -Hyperdrive.prototype.mkdir = function (name, opts, cb) { - if (typeof opts === 'function') return this.mkdir(name, null, opts) - if (typeof opts === 'number') opts = {mode: opts} - if (!opts) opts = {} - if (!cb) cb = noop - - name = unixify(name) - - var self = this - - this.ready(function (err) { - if (err) return cb(err) - if (self._checkout) return cb(new Error('Cannot write to a checkout')) - - self._lock(function (release) { - var st = { - mode: (opts.mode || DEFAULT_DMODE) | stat.IFDIR, - uid: opts.uid, - gid: opts.gid, - mtime: getTime(opts.mtime), - ctime: getTime(opts.ctime), - offset: self.content.length, - byteOffset: self.content.byteLength - } - - self.tree.put(name, st, function (err) { - release(cb, err) - }) - }) - }) -} - -Hyperdrive.prototype._statDirectory = function (name, opts, cb) { - this.tree.list(name, opts, function (err, list) { - if (name !== '/' && (err || !list.length)) return cb(err || new Error(name + ' could not be found')) - var st = stat() - st.mode = stat.IFDIR | DEFAULT_DMODE - cb(null, st) - }) -} - -Hyperdrive.prototype.access = function (name, opts, cb) { - if (typeof opts === 'function') return this.access(name, null, opts) - if (!opts) opts = {} - name = unixify(name) - this.stat(name, opts, function (err) { - cb(err) - }) -} - -Hyperdrive.prototype.exists = function (name, opts, cb) { - if (typeof opts === 'function') return this.exists(name, null, opts) - if (!opts) opts = {} - this.access(name, opts, function (err) { - cb(!err) - }) -} - -Hyperdrive.prototype.lstat = function (name, opts, cb) { - if (typeof opts === 'function') return this.lstat(name, null, opts) - if (!opts) opts = {} - var self = this - - name = unixify(name) - - this.tree.get(name, opts, function (err, st) { - if (err) return self._statDirectory(name, opts, cb) - cb(null, stat(st)) - }) -} - -Hyperdrive.prototype.stat = function (name, opts, cb) { - if (typeof opts === 'function') return this.stat(name, null, opts) - if (!opts) opts = {} - this.lstat(name, opts, cb) -} - -Hyperdrive.prototype.readdir = function (name, opts, cb) { - if (typeof opts === 'function') return this.readdir(name, null, opts) - - name = unixify(name) - - if (name === '/') return this._readdirRoot(opts, cb) // TODO: should be an option in append-tree prob - this.tree.list(name, opts, cb) -} - -Hyperdrive.prototype._readdirRoot = function (opts, cb) { - this.tree.list('/', opts, function (_, list) { - if (list) return cb(null, list) - cb(null, []) - }) -} - -Hyperdrive.prototype.unlink = function (name, cb) { - name = unixify(name) - this._del(name, cb || noop) -} - -Hyperdrive.prototype.rmdir = function (name, cb) { - if (!cb) cb = noop - - name = unixify(name) - - var self = this - - this.readdir(name, function (err, list) { - if (err) return cb(err) - if (list.length) return cb(new Error('Directory is not empty')) - self._del(name, cb) - }) -} - -Hyperdrive.prototype._del = function (name, cb) { - var self = this - - this._ensureContent(function (err) { - if (err) return cb(err) - - self._lock(function (release) { - if (!self.latest) return del(null) - self.tree.get(name, function (err, value) { - if (err) return done(err) - self.content.clear(value.offset, value.offset + value.blocks, del) - }) - - function del (err) { - if (err) return done(err) - self.tree.del(name, done) - } - - function done (err) { - release(cb, err) - } - }) - }) -} - -Hyperdrive.prototype._closeFile = function (fd, cb) { - var cursor = this._openFiles[fd - 20] - if (!cursor) return cb(new Error('Bad file descriptor')) - this._openFiles[fd - 20] = null - cursor.close(cb) -} - -Hyperdrive.prototype.close = function (fd, cb) { - if (typeof fd === 'number') return this._closeFile(fd, cb || noop) - else cb = fd - if (!cb) cb = noop - - var self = this - this.ready(function (err) { - if (err) return cb(err) - self._closed = true - self.metadata.close(function (err) { - if (!self.content) return cb(err) - self.content.close(cb) - }) - }) -} - -Hyperdrive.prototype._ensureContent = function (cb) { - var self = this - - this.ready(function (err) { - if (err) return cb(err) - if (!self.content) return self._loadIndex(cb) - cb(null) - }) -} - -Hyperdrive.prototype._loadIndex = function (cb) { - var self = this - - if (this._checkout) this._checkout._loadIndex(done) - else this.metadata.get(0, {valueEncoding: messages.Index}, done) - - function done (err, index) { - if (err) return cb(err) - if (self.content) return self.content.ready(cb) - - var keyPair = self.metadata.writable && contentKeyPair(self.metadata.secretKey) - var opts = contentOptions(self, keyPair && keyPair.secretKey) - self.content = self._checkout ? self._checkout.content : hypercore(self._storages.content, index.content, opts) - self.content.on('error', function (err) { - self.emit('error', err) - }) - self.content.ready(function (err) { - if (err) return cb(err) - self._oncontent() - cb() - }) - } -} - -Hyperdrive.prototype._open = function (cb) { - var self = this - - this.tree.ready(function (err) { - if (err) return cb(err) - self.metadata.ready(function (err) { - if (err) return cb(err) - if (self.content) return cb(null) - - self.key = self.metadata.key - self.discoveryKey = self.metadata.discoveryKey - - if (!self.metadata.writable || self._checkout) onnotwriteable() - else onwritable() - }) - }) - - function onnotwriteable () { - if (self.metadata.has(0)) return self._loadIndex(cb) - self._loadIndex(noop) - cb() - } - - function onwritable () { - var wroteIndex = self.metadata.has(0) - if (wroteIndex) return self._loadIndex(cb) - - if (!self.content) { - var keyPair = contentKeyPair(self.metadata.secretKey) - var opts = contentOptions(self, keyPair.secretKey) - self.content = hypercore(self._storages.content, keyPair.publicKey, opts) - self.content.on('error', function (err) { - self.emit('error', err) - }) - } - - self.content.ready(function () { - if (self.metadata.has(0)) return cb(new Error('Index already written')) - self.metadata.append(messages.Index.encode({type: 'hyperdrive', content: self.content.key}), cb) - }) - } -} - -function contentOptions (self, secretKey) { - return { - sparse: self.sparse || self.latest, - maxRequests: self.maxRequests, - secretKey: secretKey, - storeSecretKey: false, - indexing: self.metadata.writable && self.indexing, - storageCacheSize: self.contentStorageCacheSize - } -} - -function isObject (val) { - return !!val && typeof val !== 'string' && !Buffer.isBuffer(val) -} - -function wrap (self, storage) { - return { - metadata: function (name, opts) { - return storage.metadata(name, opts, self) - }, - content: function (name, opts) { - return storage.content(name, opts, self) - } - } -} - -function defaultStorage (self, storage, opts) { - var folder = '' - - if (typeof storage === 'object' && storage) return wrap(self, storage) - - if (typeof storage === 'string') { - folder = storage - storage = raf - } - - return { - metadata: function (name) { - return storage(path.join(folder, 'metadata', name)) - }, - content: function (name) { - return storage(path.join(folder, 'content', name)) - } - } -} - -function noop () {} - -function split (buf) { - var list = [] - for (var i = 0; i < buf.length; i += 65536) { - list.push(buf.slice(i, i + 65536)) - } - return list -} - -function getTime (date) { - if (typeof date === 'number') return date - if (!date) return Date.now() - return date.getTime() -} - -function contentKeyPair (secretKey) { - var seed = new Buffer(sodium.crypto_sign_SEEDBYTES) - var context = new Buffer('hyperdri') // 8 byte context - var keyPair = { - publicKey: new Buffer(sodium.crypto_sign_PUBLICKEYBYTES), - secretKey: new Buffer(sodium.crypto_sign_SECRETKEYBYTES) - } - - sodium.crypto_kdf_derive_from_key(seed, 1, context, secretKey) - sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed) - if (seed.fill) seed.fill(0) - - return keyPair -} diff --git a/lib/cursor.js b/lib/cursor.js deleted file mode 100644 index bd09690b..00000000 --- a/lib/cursor.js +++ /dev/null @@ -1,142 +0,0 @@ -var thunky = require('thunky') - -module.exports = Cursor - -function Cursor (archive, name, opts) { - if (!(this instanceof Cursor)) return new Cursor(archive, name, opts) - - var self = this - - this.name = name - this.opened = false - this.position = 0 - this.index = 0 - this.offset = 0 - this.open = thunky(open) - - this._content = null - this._stat = null - this._seekTo = 0 - this._seeking = true - this._start = 0 - this._end = 0 - this._range = null - this._download = !opts || opts.download !== false - - this.open() - - function open (cb) { - archive.stat(name, function (err, st) { - if (err) return cb(err) - archive._ensureContent(function (err) { - if (err) return cb(err) - if (!st.isFile()) return cb(new Error('Not a file')) - - self._content = archive.content - self._stat = st - self._start = st.offset - self._end = st.offset + st.blocks - - if (self._seekTo === 0 && self._download) { - self._range = self._content.download({start: self._start, end: self._end, linear: true}) - } - - cb(null) - }) - }) - } -} - -Cursor.prototype.seek = function (pos) { - if (pos === this.position && this._seekTo === -1) return this - this._seeking = true - this._seekTo = pos - return this -} - -Cursor.prototype._seek = function (bytes, cb) { - var self = this - - this.open(function (err) { - if (err) return cb(err) - - if (bytes < 0) bytes += self._stat.size - if (bytes < 0) bytes = 0 - if (bytes > self._stat.size) bytes = self._stat.size - - var st = self._stat - var opts = {start: self._start, end: self._end} - - if (bytes === 0) return onseek(null, self._start, 0) - if (bytes === self._stat.size) return onseek(self._end, 0) - - self._content.seek(st.byteOffset + bytes, opts, onseek) - }) - - function onseek (err, index, offset) { - if (err) return cb(err) - cb(null, bytes, index, offset) - } -} - -Cursor.prototype.next = function (cb) { - if (this._seeking) this._seekAndNext(cb) - else this._next(this.position, this.index, this.offset, cb) -} - -Cursor.prototype.close = function (cb) { - if (!cb) cb = noop - - var self = this - this.open(function (err) { - if (err) return cb(err) - if (self._range) self._content.undownload(self._range) - cb() - }) -} - -Cursor.prototype._next = function (pos, index, offset, cb) { - if (index < this._start || index >= this._end) return cb(null, null) - - var self = this - - this._content.get(this.index, function (err, data) { - if (err) return cb(err) - - if (offset) { - data = data.slice(offset) - } - - self.position = pos + data.length - self.offset = 0 - self.index++ - - cb(null, data) - }) -} - -Cursor.prototype._seekAndNext = function (cb) { - var self = this - var seekTo = this._seekTo - - this._seek(seekTo, function (err, pos, index, offset) { - if (err) return cb(err) - - if (seekTo === self._seekTo) { - self._seeking = false - self._seekTo = -1 - self.position = pos - self.index = index - self.offset = offset - - if (self._download) { - if (self._range) self._content.undownload(self._range) - self._range = self._content.download({start: self.index, end: self._end, linear: true}) - } - } - - self._next(pos, index, offset, cb) - }) -} - -function noop () {} diff --git a/lib/messages.js b/lib/messages.js deleted file mode 100644 index 452d51ac..00000000 --- a/lib/messages.js +++ /dev/null @@ -1,278 +0,0 @@ -// This file is auto generated by the protocol-buffers cli tool - -/* eslint-disable quotes */ -/* eslint-disable indent */ -/* eslint-disable no-redeclare */ - -// Remember to `npm install --save protocol-buffers-encodings` -var encodings = require('protocol-buffers-encodings') -var varint = encodings.varint -var skip = encodings.skip - -var Index = exports.Index = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Stat = exports.Stat = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -defineIndex() -defineStat() - -function defineIndex () { - var enc = [ - encodings.string, - encodings.bytes - ] - - Index.encodingLength = encodingLength - Index.encode = encode - Index.decode = decode - - function encodingLength (obj) { - var length = 0 - if (!defined(obj.type)) throw new Error("type is required") - var len = enc[0].encodingLength(obj.type) - length += 1 + len - if (defined(obj.content)) { - var len = enc[1].encodingLength(obj.content) - length += 1 + len - } - return length - } - - function encode (obj, buf, offset) { - if (!offset) offset = 0 - if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) - var oldOffset = offset - if (!defined(obj.type)) throw new Error("type is required") - buf[offset++] = 10 - enc[0].encode(obj.type, buf, offset) - offset += enc[0].encode.bytes - if (defined(obj.content)) { - buf[offset++] = 18 - enc[1].encode(obj.content, buf, offset) - offset += enc[1].encode.bytes - } - encode.bytes = offset - oldOffset - return buf - } - - function decode (buf, offset, end) { - if (!offset) offset = 0 - if (!end) end = buf.length - if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") - var oldOffset = offset - var obj = { - type: "", - content: null - } - var found0 = false - while (true) { - if (end <= offset) { - if (!found0) throw new Error("Decoded message is not valid") - decode.bytes = offset - oldOffset - return obj - } - var prefix = varint.decode(buf, offset) - offset += varint.decode.bytes - var tag = prefix >> 3 - switch (tag) { - case 1: - obj.type = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - found0 = true - break - case 2: - obj.content = enc[1].decode(buf, offset) - offset += enc[1].decode.bytes - break - default: - offset = skip(prefix & 7, buf, offset) - } - } - } -} - -function defineStat () { - var enc = [ - encodings.varint - ] - - Stat.encodingLength = encodingLength - Stat.encode = encode - Stat.decode = decode - - function encodingLength (obj) { - var length = 0 - if (!defined(obj.mode)) throw new Error("mode is required") - var len = enc[0].encodingLength(obj.mode) - length += 1 + len - if (defined(obj.uid)) { - var len = enc[0].encodingLength(obj.uid) - length += 1 + len - } - if (defined(obj.gid)) { - var len = enc[0].encodingLength(obj.gid) - length += 1 + len - } - if (defined(obj.size)) { - var len = enc[0].encodingLength(obj.size) - length += 1 + len - } - if (defined(obj.blocks)) { - var len = enc[0].encodingLength(obj.blocks) - length += 1 + len - } - if (defined(obj.offset)) { - var len = enc[0].encodingLength(obj.offset) - length += 1 + len - } - if (defined(obj.byteOffset)) { - var len = enc[0].encodingLength(obj.byteOffset) - length += 1 + len - } - if (defined(obj.mtime)) { - var len = enc[0].encodingLength(obj.mtime) - length += 1 + len - } - if (defined(obj.ctime)) { - var len = enc[0].encodingLength(obj.ctime) - length += 1 + len - } - return length - } - - function encode (obj, buf, offset) { - if (!offset) offset = 0 - if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) - var oldOffset = offset - if (!defined(obj.mode)) throw new Error("mode is required") - buf[offset++] = 8 - enc[0].encode(obj.mode, buf, offset) - offset += enc[0].encode.bytes - if (defined(obj.uid)) { - buf[offset++] = 16 - enc[0].encode(obj.uid, buf, offset) - offset += enc[0].encode.bytes - } - if (defined(obj.gid)) { - buf[offset++] = 24 - enc[0].encode(obj.gid, buf, offset) - offset += enc[0].encode.bytes - } - if (defined(obj.size)) { - buf[offset++] = 32 - enc[0].encode(obj.size, buf, offset) - offset += enc[0].encode.bytes - } - if (defined(obj.blocks)) { - buf[offset++] = 40 - enc[0].encode(obj.blocks, buf, offset) - offset += enc[0].encode.bytes - } - if (defined(obj.offset)) { - buf[offset++] = 48 - enc[0].encode(obj.offset, buf, offset) - offset += enc[0].encode.bytes - } - if (defined(obj.byteOffset)) { - buf[offset++] = 56 - enc[0].encode(obj.byteOffset, buf, offset) - offset += enc[0].encode.bytes - } - if (defined(obj.mtime)) { - buf[offset++] = 64 - enc[0].encode(obj.mtime, buf, offset) - offset += enc[0].encode.bytes - } - if (defined(obj.ctime)) { - buf[offset++] = 72 - enc[0].encode(obj.ctime, buf, offset) - offset += enc[0].encode.bytes - } - encode.bytes = offset - oldOffset - return buf - } - - function decode (buf, offset, end) { - if (!offset) offset = 0 - if (!end) end = buf.length - if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") - var oldOffset = offset - var obj = { - mode: 0, - uid: 0, - gid: 0, - size: 0, - blocks: 0, - offset: 0, - byteOffset: 0, - mtime: 0, - ctime: 0 - } - var found0 = false - while (true) { - if (end <= offset) { - if (!found0) throw new Error("Decoded message is not valid") - decode.bytes = offset - oldOffset - return obj - } - var prefix = varint.decode(buf, offset) - offset += varint.decode.bytes - var tag = prefix >> 3 - switch (tag) { - case 1: - obj.mode = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - found0 = true - break - case 2: - obj.uid = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - break - case 3: - obj.gid = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - break - case 4: - obj.size = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - break - case 5: - obj.blocks = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - break - case 6: - obj.offset = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - break - case 7: - obj.byteOffset = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - break - case 8: - obj.mtime = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - break - case 9: - obj.ctime = enc[0].decode(buf, offset) - offset += enc[0].decode.bytes - break - default: - offset = skip(prefix & 7, buf, offset) - } - } - } -} - -function defined (val) { - return val !== null && val !== undefined && (typeof val !== 'number' || !isNaN(val)) -} diff --git a/lib/stat.js b/lib/stat.js deleted file mode 100644 index 9a8702cd..00000000 --- a/lib/stat.js +++ /dev/null @@ -1,48 +0,0 @@ -// http://man7.org/linux/man-pages/man2/stat.2.html - -module.exports = Stat - -function Stat (data) { - if (!(this instanceof Stat)) return new Stat(data) - - this.dev = 0 - this.nlink = 1 - this.rdev = 0 - this.blksize = 0 - this.ino = 0 - - this.mode = data ? data.mode : 0 - this.uid = data ? data.uid : 0 - this.gid = data ? data.gid : 0 - this.size = data ? data.size : 0 - this.offset = data ? data.offset : 0 - this.byteOffset = data ? data.byteOffset : 0 - this.blocks = data ? data.blocks : 0 - this.atime = new Date(data ? data.mtime : 0) // we just set this to mtime ... - this.mtime = new Date(data ? data.mtime : 0) - this.ctime = new Date(data ? data.ctime : 0) - - this.linkname = data ? data.linkname : null -} - -Stat.IFSOCK = 49152 // 0b1100... -Stat.IFLNK = 40960 // 0b1010... -Stat.IFREG = 32768 // 0b1000... -Stat.IFBLK = 24576 // 0b0110... -Stat.IFDIR = 16384 // 0b0100... -Stat.IFCHR = 8192 // 0b0010... -Stat.IFIFO = 4096 // 0b0001... - -Stat.prototype.isSocket = check(Stat.IFSOCK) -Stat.prototype.isSymbolicLink = check(Stat.IFLNK) -Stat.prototype.isFile = check(Stat.IFREG) -Stat.prototype.isBlockDevice = check(Stat.IFBLK) -Stat.prototype.isDirectory = check(Stat.IFDIR) -Stat.prototype.isCharacterDevice = check(Stat.IFCHR) -Stat.prototype.isFIFO = check(Stat.IFIFO) - -function check (mask) { - return function () { - return (mask & this.mode) === mask - } -} diff --git a/package.json b/package.json index e1c044fd..13e7c52c 100644 --- a/package.json +++ b/package.json @@ -3,41 +3,41 @@ "version": "9.14.2", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", - "dependencies": { - "append-tree": "^2.3.5", - "duplexify": "^3.5.0", - "from2": "^2.3.0", - "hypercore": "^6.22.1", - "inherits": "^2.0.3", - "mutexify": "^1.1.0", - "protocol-buffers-encodings": "^1.1.0", - "random-access-file": "^2.0.1", - "sodium-universal": "^2.0.0", - "stream-collector": "^1.0.1", - "stream-each": "^1.2.0", - "thunky": "^1.0.2", - "uint64be": "^2.0.1", - "unixify": "^1.0.0" - }, - "devDependencies": { - "protocol-buffers": "^4.0.2", - "random-access-memory": "^2.3.0", - "standard": "^9.0.2", - "tape": "^4.6.3", - "temporary-directory": "^1.0.2" - }, "scripts": { - "test": "standard && tape test/*.js", - "protobuf": "protocol-buffers schema.proto -o lib/messages.js" + "test": "tape test/*.js" }, "repository": { "type": "git", - "url": "https://github.com/mafintosh/hyperdrive.git" + "url": "git+https://github.com/mafintosh/hyperdrive.git" }, + "keywords": [ + "hyperdrive", + "hypertrie" + ], "author": "Mathias Buus (@mafintosh)", "license": "MIT", "bugs": { "url": "https://github.com/mafintosh/hyperdrive/issues" }, - "homepage": "https://github.com/mafintosh/hyperdrive" + "homepage": "https://github.com/mafintosh/hyperdrive#readme", + "dependencies": { + "custom-error-class": "^1.0.0", + "duplexify": "^3.7.1", + "hypercore": "^6.24.0", + "hypercore-byte-stream": "^1.0.1", + "hypertrie": "^3.3.0", + "mutexify": "^1.2.0", + "pump": "^3.0.0", + "stream-collector": "^1.0.1", + "through2": "^3.0.0", + "thunky": "^1.0.3", + "unixify": "^1.0.0" + }, + "devDependencies": { + "fuzzbuzz": "^1.2.0", + "random-access-memory": "^3.1.1", + "sodium-universal": "^2.0.0", + "tape": "^4.9.2", + "temporary-directory": "^1.0.2" + } } diff --git a/schema.proto b/schema.proto deleted file mode 100644 index 6a89ff2b..00000000 --- a/schema.proto +++ /dev/null @@ -1,16 +0,0 @@ -message Index { - required string type = 1; - optional bytes content = 2; -} - -message Stat { - required uint32 mode = 1; - optional uint32 uid = 2; - optional uint32 gid = 3; - optional uint64 size = 4; - optional uint64 blocks = 5; - optional uint64 offset = 6; - optional uint64 byteOffset = 7; - optional uint64 mtime = 8; - optional uint64 ctime = 9; -} diff --git a/test/basic.js b/test/basic.js deleted file mode 100644 index 805ed098..00000000 --- a/test/basic.js +++ /dev/null @@ -1,193 +0,0 @@ -var tape = require('tape') -var sodium = require('sodium-universal') -var create = require('./helpers/create') - -tape('write and read', function (t) { - var archive = create() - - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - archive.readFile('/hello.txt', function (err, buf) { - t.error(err, 'no error') - t.same(buf, new Buffer('world')) - t.end() - }) - }) -}) - -tape('write and read (2 parallel)', function (t) { - t.plan(6) - - var archive = create() - - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - archive.readFile('/hello.txt', function (err, buf) { - t.error(err, 'no error') - t.same(buf, new Buffer('world')) - }) - }) - - archive.writeFile('/world.txt', 'hello', function (err) { - t.error(err, 'no error') - archive.readFile('/world.txt', function (err, buf) { - t.error(err, 'no error') - t.same(buf, new Buffer('hello')) - }) - }) -}) - -tape('write and read (sparse)', function (t) { - t.plan(2) - - var archive = create() - archive.on('ready', function () { - var clone = create(archive.key, {sparse: true}) - - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - var stream = clone.replicate() - stream.pipe(archive.replicate()).pipe(stream) - - var readStream = clone.createReadStream('/hello.txt') - readStream.on('data', function (data) { - t.same(data.toString(), 'world') - }) - }) - }) -}) - -tape('write and unlink', function (t) { - var archive = create() - - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - archive.unlink('/hello.txt', function (err) { - t.error(err, 'no error') - archive.readFile('/hello.txt', function (err) { - t.ok(err, 'had error') - t.end() - }) - }) - }) -}) - -tape('root is always there', function (t) { - var archive = create() - - archive.access('/', function (err) { - t.error(err, 'no error') - archive.readdir('/', function (err, list) { - t.error(err, 'no error') - t.same(list, []) - t.end() - }) - }) -}) - -tape('owner is writable', function (t) { - var archive = create() - - archive.on('ready', function () { - t.ok(archive.writable) - t.ok(archive.metadata.writable) - t.ok(archive.content.writable) - t.end() - }) -}) - -tape('provide keypair', function (t) { - var publicKey = new Buffer(sodium.crypto_sign_PUBLICKEYBYTES) - var secretKey = new Buffer(sodium.crypto_sign_SECRETKEYBYTES) - - sodium.crypto_sign_keypair(publicKey, secretKey) - - var archive = create(publicKey, {secretKey: secretKey}) - - archive.on('ready', function () { - t.ok(archive.writable) - t.ok(archive.metadata.writable) - t.ok(archive.content.writable) - t.ok(publicKey.equals(archive.key)) - - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - archive.readFile('/hello.txt', function (err, buf) { - t.error(err, 'no error') - t.same(buf, new Buffer('world')) - t.end() - }) - }) - }) -}) - -tape('download a version', function (t) { - var src = create() - src.on('ready', function () { - t.ok(src.writable) - t.ok(src.metadata.writable) - t.ok(src.content.writable) - src.writeFile('/first.txt', 'number 1', function (err) { - t.error(err, 'no error') - src.writeFile('/second.txt', 'number 2', function (err) { - t.error(err, 'no error') - src.writeFile('/third.txt', 'number 3', function (err) { - t.error(err, 'no error') - t.same(src.version, 3) - testDownloadVersion() - }) - }) - }) - }) - - function testDownloadVersion () { - var clone = create(src.key, { sparse: true }) - clone.on('content', function () { - t.same(clone.version, 3) - clone.checkout(2).download(function (err) { - t.error(err) - clone.readFile('/second.txt', { cached: true }, function (err, content) { - t.error(err, 'block not downloaded') - t.same(content && content.toString(), 'number 2', 'content does not match') - clone.readFile('/third.txt', { cached: true }, function (err, content) { - t.same(err && err.message, 'Block not downloaded') - t.end() - }) - }) - }) - }) - var stream = clone.replicate() - stream.pipe(src.replicate()).pipe(stream) - } -}) - -tape('write and read, no cache', function (t) { - var archive = create({ - metadataStorageCacheSize: 0, - contentStorageCacheSize: 0, - treeCacheSize: 0 - }) - - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - archive.readFile('/hello.txt', function (err, buf) { - t.error(err, 'no error') - t.same(buf, new Buffer('world')) - t.end() - }) - }) -}) - -tape('closing a read-only, latest clone', function (t) { - // This is just a sample key of a dead dat - var clone = create('1d5e5a628d237787afcbfec7041a16f67ba6895e7aa31500013e94ddc638328d', { - latest: true - }) - clone.on('error', function (err) { - t.fail(err) - }) - clone.close(function (err) { - t.error(err) - t.end() - }) -}) diff --git a/test/cursor.js b/test/cursor.js deleted file mode 100644 index 7a74b7f5..00000000 --- a/test/cursor.js +++ /dev/null @@ -1,87 +0,0 @@ -var tape = require('tape') -var create = require('./helpers/create') -var crypto = require('crypto') - -tape('basic cursor', function (t) { - var drive = create() - var buf = crypto.randomBytes(100 * 1024) - - drive.writeFile('/data', buf, function (err) { - t.error(err, 'no error') - - var cursor = drive.createCursor('/data') - var bufs = [] - - cursor.next(loop) - - function loop (err, next) { - t.error(err, 'no error') - - if (!next) { - t.same(Buffer.concat(bufs), buf) - t.end() - return - } else { - bufs.push(next) - } - - cursor.next(loop) - } - }) -}) - -tape('basic cursor bigger', function (t) { - var drive = create() - var buf = crypto.randomBytes(1024 * 1024) - - drive.writeFile('/data', buf, function (err) { - t.error(err, 'no error') - - var cursor = drive.createCursor('/data') - var bufs = [] - - cursor.next(loop) - - function loop (err, next) { - t.error(err, 'no error') - - if (!next) { - t.same(Buffer.concat(bufs), buf) - t.end() - return - } else { - bufs.push(next) - } - - cursor.next(loop) - } - }) -}) - -tape('cursor random access', function (t) { - var drive = create() - var buf = crypto.randomBytes(1024 * 1024) - - drive.writeFile('/data', buf, function (err) { - t.error(err, 'no error') - - var cursor = drive.createCursor('/data') - - cursor.seek(1024 * 1024 - 10).next(function (err, next) { - t.error(err, 'no error') - t.same(next, buf.slice(1024 * 1024 - 10)) - - cursor.seek(400 * 1024).next(function (err, next) { - t.error(err, 'no error') - t.same(next.slice(0, 1000), buf.slice(400 * 1024, 400 * 1024 + 1000)) - - cursor.next(function (err, nextNext) { - t.error(err, 'no error') - var offset = 400 * 1024 + next.length - t.same(nextNext, buf.slice(offset, offset + nextNext.length)) - t.end() - }) - }) - }) - }) -}) diff --git a/test/helpers/create.js b/test/helpers/create.js deleted file mode 100644 index 65dbf151..00000000 --- a/test/helpers/create.js +++ /dev/null @@ -1,6 +0,0 @@ -var ram = require('random-access-memory') -var hyperdrive = require('../../') - -module.exports = function (key, opts) { - return hyperdrive(ram, key, opts) -} diff --git a/test/stat.js b/test/stat.js deleted file mode 100644 index 2afe3369..00000000 --- a/test/stat.js +++ /dev/null @@ -1,37 +0,0 @@ -var tape = require('tape') -var create = require('./helpers/create') - -var mask = 511 // 0b111111111 - -tape('stat file', function (t) { - var archive = create() - - archive.writeFile('/foo', 'bar', {mode: 438}, function (err) { - t.error(err, 'no error') - archive.stat('/foo', function (err, st) { - t.error(err, 'no error') - t.same(st.isDirectory(), false) - t.same(st.isFile(), true) - t.same(st.mode & mask, 438) - t.same(st.size, 3) - t.same(st.offset, 0) - t.end() - }) - }) -}) - -tape('stat dir', function (t) { - var archive = create() - - archive.mkdir('/foo', function (err) { - t.error(err, 'no error') - archive.stat('/foo', function (err, st) { - t.error(err, 'no error') - t.same(st.isDirectory(), true) - t.same(st.isFile(), false) - t.same(st.mode & mask, 493) - t.same(st.offset, 0) - t.end() - }) - }) -}) diff --git a/test/storage.js b/test/storage.js deleted file mode 100644 index 3946fe6e..00000000 --- a/test/storage.js +++ /dev/null @@ -1,134 +0,0 @@ -var tape = require('tape') -var tmp = require('temporary-directory') -var create = require('./helpers/create') -var hyperdrive = require('..') - -tape('ram storage', function (t) { - var archive = create() - - archive.ready(function () { - t.ok(archive.metadata.writable, 'archive metadata is writable') - t.ok(archive.content.writable, 'archive content is writable') - t.end() - }) -}) - -tape('dir storage with resume', function (t) { - tmp(function (err, dir, cleanup) { - t.ifError(err) - var archive = hyperdrive(dir) - archive.ready(function () { - t.ok(archive.metadata.writable, 'archive metadata is writable') - t.ok(archive.content.writable, 'archive content is writable') - t.same(archive.version, 0, 'archive has version 0') - archive.close(function (err) { - t.ifError(err) - - var archive2 = hyperdrive(dir) - archive2.ready(function () { - t.ok(archive2.metadata.writable, 'archive2 metadata is writable') - t.ok(archive2.content.writable, 'archive2 content is writable') - t.same(archive2.version, 0, 'archive has version 0') - - cleanup(function (err) { - t.ifError(err) - t.end() - }) - }) - }) - }) - }) -}) - -tape('dir storage for non-writable archive', function (t) { - var src = create() - src.ready(function () { - tmp(function (err, dir, cleanup) { - t.ifError(err) - - var clone = hyperdrive(dir, src.key) - clone.on('content', function () { - t.ok(!clone.metadata.writable, 'clone metadata not writable') - t.ok(!clone.content.writable, 'clone content not writable') - t.same(clone.key, src.key, 'keys match') - cleanup(function (err) { - t.ifError(err) - t.end() - }) - }) - - var stream = clone.replicate() - stream.pipe(src.replicate()).pipe(stream) - }) - }) -}) - -tape('dir storage without permissions emits error', function (t) { - t.plan(1) - var archive = hyperdrive('/') - archive.on('error', function (err) { - t.ok(err, 'got error') - }) -}) - -tape('write and read (sparse)', function (t) { - t.plan(3) - - tmp(function (err, dir, cleanup) { - t.ifError(err) - var archive = hyperdrive(dir) - archive.on('ready', function () { - var clone = create(archive.key, {sparse: true}) - clone.on('ready', function () { - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - var stream = clone.replicate() - stream.pipe(archive.replicate()).pipe(stream) - var readStream = clone.createReadStream('/hello.txt') - readStream.on('error', function (err) { - t.error(err, 'no error') - }) - readStream.on('data', function (data) { - t.same(data.toString(), 'world') - }) - }) - }) - }) - }) -}) - -tape('sparse read/write two files', function (t) { - var archive = create() - archive.on('ready', function () { - var clone = create(archive.key, {sparse: true}) - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - archive.writeFile('/hello2.txt', 'world', function (err) { - t.error(err, 'no error') - var stream = clone.replicate() - stream.pipe(archive.replicate()).pipe(stream) - clone.metadata.update(start) - }) - }) - - function start () { - clone.stat('/hello.txt', function (err, stat) { - t.error(err, 'no error') - t.ok(stat, 'has stat') - clone.readFile('/hello.txt', function (err, data) { - t.error(err, 'no error') - t.same(data.toString(), 'world', 'data ok') - clone.stat('/hello2.txt', function (err, stat) { - t.error(err, 'no error') - t.ok(stat, 'has stat') - clone.readFile('/hello2.txt', function (err, data) { - t.error(err, 'no error') - t.same(data.toString(), 'world', 'data ok') - t.end() - }) - }) - }) - }) - } - }) -}) From 0fc77d593cb4e036025247d66a833c832b61ecd2 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 25 Jan 2019 07:54:57 -0800 Subject: [PATCH 002/108] Simple reads/writes work --- index.js | 0 lib/errors.js | 13 ++ lib/messages.js | 279 +++++++++++++++++++++++++++++++++++++++++ lib/stat.js | 77 ++++++++++++ schema.proto | 16 +++ test/basic.js | 17 +++ test/creation.js | 15 +++ test/helpers/create.js | 6 + 8 files changed, 423 insertions(+) create mode 100644 index.js create mode 100644 lib/errors.js create mode 100644 lib/messages.js create mode 100644 lib/stat.js create mode 100644 schema.proto create mode 100644 test/basic.js create mode 100644 test/creation.js create mode 100644 test/helpers/create.js diff --git a/index.js b/index.js new file mode 100644 index 00000000..e69de29b diff --git a/lib/errors.js b/lib/errors.js new file mode 100644 index 00000000..e59d637d --- /dev/null +++ b/lib/errors.js @@ -0,0 +1,13 @@ +const CustomError = require('custom-error-class') + +class FileNotFound extends CustomError { + constructor (fileName) { + super(`File '${fileName}' not found.`) + this.code = 'ENOENT' + this.errno = 2 + } +} + +module.exports = { + FileNotFound +} diff --git a/lib/messages.js b/lib/messages.js new file mode 100644 index 00000000..9fc819d2 --- /dev/null +++ b/lib/messages.js @@ -0,0 +1,279 @@ +// This file is auto generated by the protocol-buffers compiler + +/* eslint-disable quotes */ +/* eslint-disable indent */ +/* eslint-disable no-redeclare */ +/* eslint-disable camelcase */ + +// Remember to `npm install --save protocol-buffers-encodings` +var encodings = require('protocol-buffers-encodings') +var varint = encodings.varint +var skip = encodings.skip + +var Index = exports.Index = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Stat = exports.Stat = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +defineIndex() +defineStat() + +function defineIndex () { + var enc = [ + encodings.string, + encodings.bytes + ] + + Index.encodingLength = encodingLength + Index.encode = encode + Index.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.type)) throw new Error("type is required") + var len = enc[0].encodingLength(obj.type) + length += 1 + len + if (defined(obj.content)) { + var len = enc[1].encodingLength(obj.content) + length += 1 + len + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.type)) throw new Error("type is required") + buf[offset++] = 10 + enc[0].encode(obj.type, buf, offset) + offset += enc[0].encode.bytes + if (defined(obj.content)) { + buf[offset++] = 18 + enc[1].encode(obj.content, buf, offset) + offset += enc[1].encode.bytes + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + type: "", + content: null + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.type = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + found0 = true + break + case 2: + obj.content = enc[1].decode(buf, offset) + offset += enc[1].decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defineStat () { + var enc = [ + encodings.varint + ] + + Stat.encodingLength = encodingLength + Stat.encode = encode + Stat.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.mode)) throw new Error("mode is required") + var len = enc[0].encodingLength(obj.mode) + length += 1 + len + if (defined(obj.uid)) { + var len = enc[0].encodingLength(obj.uid) + length += 1 + len + } + if (defined(obj.gid)) { + var len = enc[0].encodingLength(obj.gid) + length += 1 + len + } + if (defined(obj.size)) { + var len = enc[0].encodingLength(obj.size) + length += 1 + len + } + if (defined(obj.blocks)) { + var len = enc[0].encodingLength(obj.blocks) + length += 1 + len + } + if (defined(obj.offset)) { + var len = enc[0].encodingLength(obj.offset) + length += 1 + len + } + if (defined(obj.byteOffset)) { + var len = enc[0].encodingLength(obj.byteOffset) + length += 1 + len + } + if (defined(obj.mtime)) { + var len = enc[0].encodingLength(obj.mtime) + length += 1 + len + } + if (defined(obj.ctime)) { + var len = enc[0].encodingLength(obj.ctime) + length += 1 + len + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.mode)) throw new Error("mode is required") + buf[offset++] = 8 + enc[0].encode(obj.mode, buf, offset) + offset += enc[0].encode.bytes + if (defined(obj.uid)) { + buf[offset++] = 16 + enc[0].encode(obj.uid, buf, offset) + offset += enc[0].encode.bytes + } + if (defined(obj.gid)) { + buf[offset++] = 24 + enc[0].encode(obj.gid, buf, offset) + offset += enc[0].encode.bytes + } + if (defined(obj.size)) { + buf[offset++] = 32 + enc[0].encode(obj.size, buf, offset) + offset += enc[0].encode.bytes + } + if (defined(obj.blocks)) { + buf[offset++] = 40 + enc[0].encode(obj.blocks, buf, offset) + offset += enc[0].encode.bytes + } + if (defined(obj.offset)) { + buf[offset++] = 48 + enc[0].encode(obj.offset, buf, offset) + offset += enc[0].encode.bytes + } + if (defined(obj.byteOffset)) { + buf[offset++] = 56 + enc[0].encode(obj.byteOffset, buf, offset) + offset += enc[0].encode.bytes + } + if (defined(obj.mtime)) { + buf[offset++] = 64 + enc[0].encode(obj.mtime, buf, offset) + offset += enc[0].encode.bytes + } + if (defined(obj.ctime)) { + buf[offset++] = 72 + enc[0].encode(obj.ctime, buf, offset) + offset += enc[0].encode.bytes + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + mode: 0, + uid: 0, + gid: 0, + size: 0, + blocks: 0, + offset: 0, + byteOffset: 0, + mtime: 0, + ctime: 0 + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.mode = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + found0 = true + break + case 2: + obj.uid = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + break + case 3: + obj.gid = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + break + case 4: + obj.size = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + break + case 5: + obj.blocks = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + break + case 6: + obj.offset = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + break + case 7: + obj.byteOffset = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + break + case 8: + obj.mtime = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + break + case 9: + obj.ctime = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defined (val) { + return val !== null && val !== undefined && (typeof val !== 'number' || !isNaN(val)) +} diff --git a/lib/stat.js b/lib/stat.js new file mode 100644 index 00000000..65e270eb --- /dev/null +++ b/lib/stat.js @@ -0,0 +1,77 @@ +// http://man7.org/linux/man-pages/man2/stat.2.html + +var DEFAULT_FMODE = (4 | 2 | 0) << 6 | ((4 | 0 | 0) << 3) | (4 | 0 | 0) // rw-r--r-- +var DEFAULT_DMODE = (4 | 2 | 1) << 6 | ((4 | 0 | 1) << 3) | (4 | 0 | 1) // rwxr-xr-x + +class Stat { + constructor (data) { + this.dev = 0 + this.nlink = 1 + this.rdev = 0 + this.blksize = 0 + this.ino = 0 + + this.mode = (data && data.mode) || 0 + this.uid = (data && data.uid) || 0 + this.gid = (data && data.gid) || 0 + this.size = (data && data.size) || 0 + this.offset = (data && data.offset) || 0 + this.byteOffset = (data && data.byteOffset) || 0 + this.blocks = (data && data.blocks) || 0 + this.atime = data && data.atime ? getTime(data.atime) : 0 // we just set this to mtime ... + this.mtime = data && data.mtime ? getTime(data.mtime) : 0 + this.ctime = data && data.ctime? getTime(data.ctime) : 0 + this.linkname = (data && data.linkname) || null + } + + _check (mask) { + return (mask & this.mode) === mask + } + + isSocket () { + this._check(Stat.IFSOCK) + } + isSymbolicLink () { + this._check(Stat.IFLNK) + } + isFile () { + this._check(Stat.IFREG) + } + isBlockDevice () { + this._check(Stat.IFBLK) + } + isDirectory () { + this._check(Stat.IFDIR) + } + isCharacterDevice () { + this._check(Stat.IFCHR) + } + isFIFO () { + this._check(Stat.IFIFO) + } +} + +Stat.file = function (data) { + data.mode = data.mode || DEFAULT_FMODE | Stat.IFREG + return new Stat(data) +} +Stat.directory = function (data) { + data.mode = data.mode || DEFAULT_DMODE | Stat.IFDIR + return new Stat(data) +} + +Stat.IFSOCK = 0b1100 << 12 +Stat.IFLNK = 0b1010 << 12 +Stat.IFREG = 0b1000 << 12 +Stat.IFBLK = 0b0110 << 12 +Stat.IFDIR = 0b0100 << 12 +Stat.IFCHR = 0b0010 << 12 +Stat.IFIFO = 0b0001 << 12 + +function getTime (date) { + if (typeof date === 'number') return date + if (!date) return Date.now() + return date.getTime() +} + +module.exports = Stat diff --git a/schema.proto b/schema.proto new file mode 100644 index 00000000..6a89ff2b --- /dev/null +++ b/schema.proto @@ -0,0 +1,16 @@ +message Index { + required string type = 1; + optional bytes content = 2; +} + +message Stat { + required uint32 mode = 1; + optional uint32 uid = 2; + optional uint32 gid = 3; + optional uint64 size = 4; + optional uint64 blocks = 5; + optional uint64 offset = 6; + optional uint64 byteOffset = 7; + optional uint64 mtime = 8; + optional uint64 ctime = 9; +} diff --git a/test/basic.js b/test/basic.js new file mode 100644 index 00000000..789e1de9 --- /dev/null +++ b/test/basic.js @@ -0,0 +1,17 @@ +var tape = require('tape') +var sodium = require('sodium-universal') +var create = require('./helpers/create') + +tape('write and read', function (t) { + var archive = create() + + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + archive.readFile('/hello.txt', function (err, buf) { + t.error(err, 'no error') + t.same(buf, Buffer.from('world')) + t.end() + }) + }) +}) + diff --git a/test/creation.js b/test/creation.js new file mode 100644 index 00000000..a4a3e135 --- /dev/null +++ b/test/creation.js @@ -0,0 +1,15 @@ +var tape = require('tape') +var create = require('./helpers/create') + +tape('owner is writable', function (t) { + var archive = create() + + archive.on('ready', function () { + t.ok(archive.writable) + t.ok(archive.metadataFeed.writable) + t.ok(archive.content.writable) + t.end() + }) +}) + + diff --git a/test/helpers/create.js b/test/helpers/create.js new file mode 100644 index 00000000..7803e35b --- /dev/null +++ b/test/helpers/create.js @@ -0,0 +1,6 @@ +var ram = require('random-access-memory') +var Hyperdrive = require('../../') + +module.exports = function (key, opts) { + return new Hyperdrive(ram, key, opts) +} From 79e282ea587f296095024b6d0739ca740723e7d6 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 25 Jan 2019 08:02:13 -0800 Subject: [PATCH 003/108] Forgot index file --- index.js | 358 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 358 insertions(+) diff --git a/index.js b/index.js index e69de29b..cb239e96 100644 --- a/index.js +++ b/index.js @@ -0,0 +1,358 @@ +const path = require('path') +const { EventEmitter } = require('events') + +const collect = require('stream-collector') +const thunky = require('thunky') +const unixify = require('unixify') +const raf = require('random-access-file') +const mutexify = require('mutexify') +const duplexify = require('duplexify') +const sodium = require('sodium-universal') + +const hypercore = require('hypercore') +const hypertrie = require('hypertrie') +const coreByteStream = require('hypercore-byte-stream') + +const Stat = require('./lib/stat') +const errors = require('./lib/errors') +const messages = require('./lib/messages') + +module.exports = class Hyperdrive extends EventEmitter { + constructor (storage, key, opts) { + super() + + if (isObject(key)) { + opts = key + key = null + } + if (!opts) opts = {} + + this.key = null + this.discoveryKey = null + this.live = true + this.latest = !!opts.latest + + this._storages = defaultStorage(this, storage, opts) + + this.metadataFeed = hypercore(this._storages.metadata, key, { + secretKey: opts.secretKey, + sparse: opts.sparseMetadata, + createIfMissing: opts.createIfMissing, + storageCacheSize: opts.metadataStorageCacheSize, + valueEncoding: 'binary' + }) + this.trie = opts.metadata + this.content = opts.content || null + this.storage = storage + + this._lock = mutexify() + + this.ready = thunky(this._ready.bind(this)) + this.ready(onready) + + var self = this + + function onready (err) { + if (err) return onerror(err) + self.emit('ready') + /* + if (self.latest && !self.metadata.writable) { + self._trackLatest(function (err) { + if (self._closed) return + onerror(err) + }) + } + */ + } + + } + + _ready (cb) { + var self = this + + self.metadataFeed.ready(function (err) { + if (err) return cb(err) + + /** + * If the metadata feed is writable: + * If the metadata feed has length 0, then the trie should be initialized with the content feed key as metadata. + * Else, initialize the trie without metadata and load the content feed key from the header. + * If the metadata feed is readable: + * Initialize the trie without metadata and load the content feed key from the header. + */ + if (self.metadataFeed.writable && !self.metadataFeed.length) { + initialize() + } else { + restore() + } + }) + + function initialize () { + var keyPair = contentKeyPair(self.metadataFeed.secretKey) + var opts = contentOptions(self, keyPair.secretKey) + self.content = hypercore(self._storages.content, keyPair.publicKey, opts) + self.content.on('error', function (err) { + self.emit('error', err) + }) + self.content.ready(function (err) { + if (err) return cb(err) + + self.trie = hypertrie(null, { + feed: self.metadataFeed, + metadata: self.content.key, + valueEncoding: messages.Stat + }) + + self.trie.ready(function (err) { + if (err) return cb(err) + return done(null) + }) + }) + } + + function restore () { + self.trie = hypertrie(null, { + feed: self.metadataFeed + }) + self.trie.ready(function (err) { + if (err) return cb(err) + + self.trie.metadata(function (err, contentKey) { + if (err) return cb(err) + + self.content = hypercore(self._storages.content, contentKey, contentOptions(self, null)) + self.content.ready(done) + }) + }) + } + + function done (err) { + if (err) return cb(err) + + self.key = self.metadataFeed.key + self.discoveryKey = self.metadataFeed.discoveryKey + + self.metadataFeed.on('update', update) + self.metadataFeed.on('error', onerror) + self.content.on('error', onerror) + + self.writable = self.metadataFeed.writable && self.content.writable + + self.emit('content') + + return cb(null) + } + + function onerror (err) { + if (err) self.emit('error', err) + } + + function update () { + self.emit('update') + } + } + + createReadStream (name, opts) { + if (!opts) opts = {} + + name = unixify(name) + + var stream = coreByteStream({ + ...opts, + highWaterMark: opts.highWaterMark || 64 * 1024 + }) + + this.ready(err => { + if (err) return stream.destroy(err) + + this.trie.get(name, (err, st) => { + if (err) return stream.destroy(err) + if (!st) return stream.destroy(new errors.FileNotFound(name)) + + st = st.value + + var byteOffset = (opts.start) ? st.byteOffset + opts.start : st.byteOffset + var byteLength = (opts.start) ? st.size - opts.start : st.size + + stream.start({ + feed: this.content, + blockOffset: st.offset, + blockLength: st.blocks, + byteOffset, + byteLength + }) + }) + }) + + + return stream + } + + createWriteStream (name, opts) { + if (!opts) opts = {} + + name = unixify(name) + + var self = this + var proxy = duplexify() + var release = null + proxy.setReadable(false) + + // TODO: support piping through a "split" stream like rabin + + this.ready(err => { + if (err) return proxy.destroy(err) + this._lock(_release => { + release = _release + this.trie.get(name, (err, st) => { + if (err) return proxy.destroy(err) + if (!st) return append(null) + this.content.clear(st.offset, st.offset + st.blocks, append) + }) + }) + }) + + return proxy + + function append (err) { + if (err) proxy.destroy(err) + if (proxy.destroyed) return release() + + // No one should mutate the content other than us + var byteOffset = self.content.byteLength + var offset = self.content.length + + self.emit('appending', name, opts) + + // TODO: revert the content feed if this fails!!!! (add an option to the write stream for this (atomic: true)) + var stream = self.content.createWriteStream() + + proxy.on('close', done) + proxy.on('finish', done) + + proxy.setWritable(stream) + proxy.on('prefinish', function () { + var st = Stat.file({ + ...opts, + size: self.content.byteLength - byteOffset, + blocks: self.content.length - offset, + offset: offset, + byteOffset: byteOffset, + }) + + proxy.cork() + self.trie.put(name, st, function (err) { + if (err) return proxy.destroy(err) + self.emit('append', name, opts) + proxy.uncork() + }) + }) + } + + function done () { + proxy.removeListener('close', done) + proxy.removeListener('finish', done) + release() + } + } + + readFile (name, opts, cb) { + if (typeof opts === 'function') return this.readFile(name, null, opts) + if (typeof opts === 'string') opts = {encoding: opts} + if (!opts) opts = {} + + name = unixify(name) + + collect(this.createReadStream(name, opts), function (err, bufs) { + if (err) return cb(err) + var buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs) + cb(null, opts.encoding && opts.encoding !== 'binary' ? buf.toString(opts.encoding) : buf) + }) + } + + writeFile (name, buf, opts, cb) { + if (typeof opts === 'function') return this.writeFile(name, buf, null, opts) + if (typeof opts === 'string') opts = {encoding: opts} + if (!opts) opts = {} + if (typeof buf === 'string') buf = new Buffer(buf, opts.encoding || 'utf-8') + if (!cb) cb = noop + + name = unixify(name) + + var bufs = split(buf) // split the input incase it is a big buffer. + var stream = this.createWriteStream(name, opts) + stream.on('error', cb) + stream.on('finish', cb) + for (var i = 0; i < bufs.length; i++) stream.write(bufs[i]) + stream.end() + } +} + +function isObject (val) { + return !!val && typeof val !== 'string' && !Buffer.isBuffer(val) +} + +function wrap (self, storage) { + return { + metadata: function (name, opts) { + return storage.metadata(name, opts, self) + }, + content: function (name, opts) { + return storage.content(name, opts, self) + } + } +} + +function defaultStorage (self, storage, opts) { + var folder = '' + + if (typeof storage === 'object' && storage) return wrap(self, storage) + + if (typeof storage === 'string') { + folder = storage + storage = raf + } + + return { + metadata: function (name) { + return storage(path.join(folder, 'metadata', name)) + }, + content: function (name) { + return storage(path.join(folder, 'content', name)) + } + } +} + +function contentOptions (self, secretKey) { + return { + sparse: self.sparse || self.latest, + maxRequests: self.maxRequests, + secretKey: secretKey, + storeSecretKey: false, + indexing: self.metadataFeed.writable && self.indexing, + storageCacheSize: self.contentStorageCacheSize + } +} + +function contentKeyPair (secretKey) { + var seed = new Buffer(sodium.crypto_sign_SEEDBYTES) + var context = new Buffer('hyperdri') // 8 byte context + var keyPair = { + publicKey: new Buffer(sodium.crypto_sign_PUBLICKEYBYTES), + secretKey: new Buffer(sodium.crypto_sign_SECRETKEYBYTES) + } + + sodium.crypto_kdf_derive_from_key(seed, 1, context, secretKey) + sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed) + if (seed.fill) seed.fill(0) + + return keyPair +} + +function split (buf) { + var list = [] + for (var i = 0; i < buf.length; i += 65536) { + list.push(buf.slice(i, i + 65536)) + } + return list +} From 3c8a6a24935f0ab4dc3e06559d4b72b5ccd1c72c Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 26 Jan 2019 09:56:49 -0800 Subject: [PATCH 004/108] Added stat, dir, and checkout methods. Need to fix initialization for checkout --- index.js | 201 +++++++++++++++++++++++++++++++++++++++++------ lib/errors.js | 11 ++- test/basic.js | 182 ++++++++++++++++++++++++++++++++++++++++++ test/creation.js | 2 +- test/stat.js | 37 +++++++++ 5 files changed, 408 insertions(+), 25 deletions(-) create mode 100644 test/stat.js diff --git a/index.js b/index.js index cb239e96..a2495306 100644 --- a/index.js +++ b/index.js @@ -17,7 +17,7 @@ const Stat = require('./lib/stat') const errors = require('./lib/errors') const messages = require('./lib/messages') -module.exports = class Hyperdrive extends EventEmitter { +class Hyperdrive extends EventEmitter { constructor (storage, key, opts) { super() @@ -34,17 +34,18 @@ module.exports = class Hyperdrive extends EventEmitter { this._storages = defaultStorage(this, storage, opts) - this.metadataFeed = hypercore(this._storages.metadata, key, { + this.metadataFeed = opts.metatadataFeed || hypercore(this._storages.metadata, key, { secretKey: opts.secretKey, sparse: opts.sparseMetadata, createIfMissing: opts.createIfMissing, storageCacheSize: opts.metadataStorageCacheSize, valueEncoding: 'binary' }) - this.trie = opts.metadata - this.content = opts.content || null + this.trie = opts.trie + this.contentFeed = opts.contentFeed || null this.storage = storage + this._checkout = opts._checkout this._lock = mutexify() this.ready = thunky(this._ready.bind(this)) @@ -64,7 +65,14 @@ module.exports = class Hyperdrive extends EventEmitter { } */ } + } + + get version () { + return this._checkout ? this.trie.version : (this.metadataFeed.length ? this.metadataFeed.length - 1 : 0) + } + get writable () { + return this.metadataFeed.writable && this.contentFeed.writable } _ready (cb) { @@ -90,16 +98,16 @@ module.exports = class Hyperdrive extends EventEmitter { function initialize () { var keyPair = contentKeyPair(self.metadataFeed.secretKey) var opts = contentOptions(self, keyPair.secretKey) - self.content = hypercore(self._storages.content, keyPair.publicKey, opts) - self.content.on('error', function (err) { + self.contentFeed = hypercore(self._storages.content, keyPair.publicKey, opts) + self.contentFeed.on('error', function (err) { self.emit('error', err) }) - self.content.ready(function (err) { + self.contentFeed.ready(function (err) { if (err) return cb(err) - self.trie = hypertrie(null, { + self.trie = self.trie || hypertrie(null, { feed: self.metadataFeed, - metadata: self.content.key, + metadata: self.contentFeed.key, valueEncoding: messages.Stat }) @@ -117,11 +125,11 @@ module.exports = class Hyperdrive extends EventEmitter { self.trie.ready(function (err) { if (err) return cb(err) - self.trie.metadata(function (err, contentKey) { + self.trie.getMetadata(function (err, contentKey) { if (err) return cb(err) - self.content = hypercore(self._storages.content, contentKey, contentOptions(self, null)) - self.content.ready(done) + self.contentFeed = hypercore(self._storages.content, contentKey, contentOptions(self, null)) + self.contentFeed.ready(done) }) }) } @@ -134,9 +142,7 @@ module.exports = class Hyperdrive extends EventEmitter { self.metadataFeed.on('update', update) self.metadataFeed.on('error', onerror) - self.content.on('error', onerror) - - self.writable = self.metadataFeed.writable && self.content.writable + self.contentFeed.on('error', onerror) self.emit('content') @@ -175,7 +181,7 @@ module.exports = class Hyperdrive extends EventEmitter { var byteLength = (opts.start) ? st.size - opts.start : st.size stream.start({ - feed: this.content, + feed: this.contentFeed, blockOffset: st.offset, blockLength: st.blocks, byteOffset, @@ -184,7 +190,6 @@ module.exports = class Hyperdrive extends EventEmitter { }) }) - return stream } @@ -207,7 +212,7 @@ module.exports = class Hyperdrive extends EventEmitter { this.trie.get(name, (err, st) => { if (err) return proxy.destroy(err) if (!st) return append(null) - this.content.clear(st.offset, st.offset + st.blocks, append) + this.contentFeed.clear(st.offset, st.offset + st.blocks, append) }) }) }) @@ -219,13 +224,13 @@ module.exports = class Hyperdrive extends EventEmitter { if (proxy.destroyed) return release() // No one should mutate the content other than us - var byteOffset = self.content.byteLength - var offset = self.content.length + var byteOffset = self.contentFeed.byteLength + var offset = self.contentFeed.length self.emit('appending', name, opts) // TODO: revert the content feed if this fails!!!! (add an option to the write stream for this (atomic: true)) - var stream = self.content.createWriteStream() + var stream = self.contentFeed.createWriteStream() proxy.on('close', done) proxy.on('finish', done) @@ -234,8 +239,8 @@ module.exports = class Hyperdrive extends EventEmitter { proxy.on('prefinish', function () { var st = Stat.file({ ...opts, - size: self.content.byteLength - byteOffset, - blocks: self.content.length - offset, + size: self.contentFeed.byteLength - byteOffset, + blocks: self.contentFeed.length - offset, offset: offset, byteOffset: byteOffset, }) @@ -286,8 +291,156 @@ module.exports = class Hyperdrive extends EventEmitter { for (var i = 0; i < bufs.length; i++) stream.write(bufs[i]) stream.end() } + + mkdir (name, opts, cb) { + if (typeof opts === 'function') return this.mkdir(name, null, opts) + if (typeof opts === 'number') opts = {mode: opts} + if (!opts) opts = {} + if (!cb) cb = noop + + name = unixify(name) + + this._lock(release => { + var st = Stat.directory({ + ...opts, + offset: this.contentFeed.length, + byteOffset: this.contentFeed.byteLength + }) + this.trie.put(name, st, err => { + release(cb, err) + }) + }) + } + + _statDirectory (name, opts, cb) { + var ite = this.trie.iterator(name) + ite.next((err, st) => { + if (err) return cb(err) + if (name !== '/' && !st) return cb(new errors.FileNotFound(name)) + let st = Stat.directory() + return cb(null, st) + }) + } + + lstat (name, opts, cb) { + if (typeof opts === 'function') return this.lstat(name, null, opts) + if (!opts) opts = {} + name = unixify(name) + + this.trie.get(name, opts, (err, st) => { + if (err) return this._statDirectory(name, opts, cb) + cb(null, new Stat(st)) + }) + } + + stat (name, opts, cb) { + if (typeof opts === 'function') return this.stat(name, null, opts) + if (!opts) opts = {} + + this.lstat(name, opts, cb) + } + + access (name, opts, cb) { + if (typeof opts === 'function') return this.access(name, null, opts) + if (!opts) opts = {} + name = unixify(name) + + this.stat(name, opts, err => { + cb(err) + }) + } + + exists (name, opts, cb) { + if (typeof opts === 'function') return this.exists(name, null, opts) + if (!opts) opts = {} + + this.access(name, opts, err => { + cb(!err) + }) + } + + readdir (name, opts, cb) { + if (typeof opts === 'function') return this.readdir(name, null, opts) + name = unixify(name) + + return this.trie.list(name, opts, cb) + } + + _del (name, cb) { + var _release = null + + this.ready(err => { + if (err) return cb(err) + this._lock(release => { + _release = release + this.trie.get(name, (err, st) => { + if (err) return done(err) + self.contentFeed.clear(st.offset, st.offset + st.blocks, del) + }) + }) + }) + + function del (err) { + if (err) return done(err) + self.trie.del(name, done) + } + + function done (err) { + _release(cb, err) + } + } + + unlink (name, cb) { + name = unixify(name) + + this._del(name, cb || noop) + } + + rmdir (name, cb) { + if (!cb) cb = noop + name = unixify(name) + + this.readdir(name, function (err, list) { + if (err) return cb(err) + if (list.length) return cb(new errors.DirectoryNotEmpty(name)) + self._del(name, cb) + }) + } + + replicate (opts) { + if (!opts) opts = {} + opts.expectedFeeds = 2 + + var stream = this.metadataFeed.replicate(opts) + + this.ready(err => { + if (err) return stream.destroy(err) + if (stream.destroyed) return + this.contentFeed.replicate({ + live: opts.live, + download: opts.download, + upload: opts.upload, + stream: stream + }) + }) + + return stream + } + + checkout (version, opts) { + var versionedTrie = this.trie.checkout(version) + opts = { + ...opts, + metadataFeed: this.metadataFeed, + contentFeed: this.contentFeed, + trie: versionedTrie, + } + return new Hyperdrive(null, null, opts) + } } +module.exports = Hyperdrive + function isObject (val) { return !!val && typeof val !== 'string' && !Buffer.isBuffer(val) } @@ -356,3 +509,5 @@ function split (buf) { } return list } + +function noop () {} diff --git a/lib/errors.js b/lib/errors.js index e59d637d..108325b8 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -8,6 +8,15 @@ class FileNotFound extends CustomError { } } +class DirectoryNotEmpty extends CustomError { + constructor (dirName) { + super(`Directory '${dirName}' is not empty.`) + this.code = 'ENOTEMPTY' + this.errno = 39 + } +} + module.exports = { - FileNotFound + FileNotFound, + DirectoryNotEmpty } diff --git a/test/basic.js b/test/basic.js index 789e1de9..0f09896b 100644 --- a/test/basic.js +++ b/test/basic.js @@ -15,3 +15,185 @@ tape('write and read', function (t) { }) }) +tape('write and read, with encoding', function (t) { + var archive = create() + + archive.writeFile('/hello.txt', 'world', { encoding: 'utf8' }, function (err) { + t.error(err, 'no error') + archive.readFile('/hello.txt', { encoding: 'utf8' }, function (err, str) { + t.error(err, 'no error') + t.same(str, 'world') + t.end() + }) + }) +}) + +tape('write and read (2 parallel)', function (t) { + t.plan(6) + + var archive = create() + + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + archive.readFile('/hello.txt', function (err, buf) { + t.error(err, 'no error') + t.same(buf, new Buffer('world')) + }) + }) + + archive.writeFile('/world.txt', 'hello', function (err) { + t.error(err, 'no error') + archive.readFile('/world.txt', function (err, buf) { + t.error(err, 'no error') + t.same(buf, new Buffer('hello')) + }) + }) +}) + +tape.skip('write and read (sparse)', function (t) { + t.plan(2) + + var archive = create() + archive.on('ready', function () { + var clone = create(archive.key, {sparse: true}) + + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + var stream = clone.replicate() + stream.pipe(archive.replicate()).pipe(stream) + + var readStream = clone.createReadStream('/hello.txt') + readStream.on('data', function (data) { + t.same(data.toString(), 'world') + }) + }) + }) +}) + +tape.skip('write and unlink', function (t) { + var archive = create() + + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + archive.unlink('/hello.txt', function (err) { + t.error(err, 'no error') + archive.readFile('/hello.txt', function (err) { + t.ok(err, 'had error') + t.end() + }) + }) + }) +}) + +tape.skip('root is always there', function (t) { + var archive = create() + + archive.access('/', function (err) { + t.error(err, 'no error') + archive.readdir('/', function (err, list) { + t.error(err, 'no error') + t.same(list, []) + t.end() + }) + }) +}) + +tape.skip('provide keypair', function (t) { + var publicKey = new Buffer(sodium.crypto_sign_PUBLICKEYBYTES) + var secretKey = new Buffer(sodium.crypto_sign_SECRETKEYBYTES) + + sodium.crypto_sign_keypair(publicKey, secretKey) + + var archive = create(publicKey, {secretKey: secretKey}) + + archive.on('ready', function () { + t.ok(archive.writable) + t.ok(archive.metadataFeed.writable) + t.ok(archive.contentFeek.writable) + t.ok(publicKey.equals(archive.key)) + + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + archive.readFile('/hello.txt', function (err, buf) { + t.error(err, 'no error') + t.same(buf, new Buffer('world')) + t.end() + }) + }) + }) +}) + +tape('download a version', function (t) { + var src = create() + src.on('ready', function () { + t.ok(src.writable) + t.ok(src.metadataFeed.writable) + t.ok(src.contentFeed.writable) + src.writeFile('/first.txt', 'number 1', function (err) { + t.error(err, 'no error') + src.writeFile('/second.txt', 'number 2', function (err) { + t.error(err, 'no error') + src.writeFile('/third.txt', 'number 3', function (err) { + t.error(err, 'no error') + t.same(src.version, 3) + testDownloadVersion() + }) + }) + }) + }) + + function testDownloadVersion () { + var clone = create(src.key, { sparse: true }) + clone.on('content', function () { + t.same(clone.version, 3) + clone.checkout(2).download(function (err) { + t.error(err) + clone.readFile('/second.txt', { cached: true }, function (err, content) { + t.error(err, 'block not downloaded') + t.same(content && content.toString(), 'number 2', 'content does not match') + clone.readFile('/third.txt', { cached: true }, function (err, content) { + t.same(err && err.message, 'Block not downloaded') + t.end() + }) + }) + }) + }) + var stream = clone.replicate() + stream.pipe(src.replicate()).pipe(stream) + } +}) + +tape.skip('write and read, no cache', function (t) { + var archive = create({ + metadataStorageCacheSize: 0, + contentStorageCacheSize: 0, + treeCacheSize: 0 + }) + + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + archive.readFile('/hello.txt', function (err, buf) { + t.error(err, 'no error') + t.same(buf, new Buffer('world')) + t.end() + }) + }) +}) + +tape.skip('closing a read-only, latest clone', function (t) { + // This is just a sample key of a dead dat + var clone = create('1d5e5a628d237787afcbfec7041a16f67ba6895e7aa31500013e94ddc638328d', { + latest: true + }) + clone.on('error', function (err) { + t.fail(err) + }) + clone.close(function (err) { + t.error(err) + t.end() + }) +}) + + + + diff --git a/test/creation.js b/test/creation.js index a4a3e135..a9fb828a 100644 --- a/test/creation.js +++ b/test/creation.js @@ -7,7 +7,7 @@ tape('owner is writable', function (t) { archive.on('ready', function () { t.ok(archive.writable) t.ok(archive.metadataFeed.writable) - t.ok(archive.content.writable) + t.ok(archive.contentFeed.writable) t.end() }) }) diff --git a/test/stat.js b/test/stat.js new file mode 100644 index 00000000..2afe3369 --- /dev/null +++ b/test/stat.js @@ -0,0 +1,37 @@ +var tape = require('tape') +var create = require('./helpers/create') + +var mask = 511 // 0b111111111 + +tape('stat file', function (t) { + var archive = create() + + archive.writeFile('/foo', 'bar', {mode: 438}, function (err) { + t.error(err, 'no error') + archive.stat('/foo', function (err, st) { + t.error(err, 'no error') + t.same(st.isDirectory(), false) + t.same(st.isFile(), true) + t.same(st.mode & mask, 438) + t.same(st.size, 3) + t.same(st.offset, 0) + t.end() + }) + }) +}) + +tape('stat dir', function (t) { + var archive = create() + + archive.mkdir('/foo', function (err) { + t.error(err, 'no error') + archive.stat('/foo', function (err, st) { + t.error(err, 'no error') + t.same(st.isDirectory(), true) + t.same(st.isFile(), false) + t.same(st.mode & mask, 493) + t.same(st.offset, 0) + t.end() + }) + }) +}) From f04561a1421bb8c31e85743906c4b2686e9cc6ac Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 28 Jan 2019 07:14:06 -0800 Subject: [PATCH 005/108] Stat + storage tests pass --- index.js | 175 +++++++++++++++++++++++++++++++++--------------- lib/stat.js | 20 +++--- test/basic.js | 19 +++--- test/storage.js | 134 ++++++++++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 74 deletions(-) create mode 100644 test/storage.js diff --git a/index.js b/index.js index a2495306..3d7c769a 100644 --- a/index.js +++ b/index.js @@ -46,15 +46,19 @@ class Hyperdrive extends EventEmitter { this.storage = storage this._checkout = opts._checkout + this._contentOpts = null this._lock = mutexify() this.ready = thunky(this._ready.bind(this)) - this.ready(onready) + this.contentReady = thunky(this._contentReady.bind(this)) + + this.ready(onReady) + this.contentReady(onContentReady) var self = this - function onready (err) { - if (err) return onerror(err) + function onReady (err) { + if (err) return self.emit('error', err) self.emit('ready') /* if (self.latest && !self.metadata.writable) { @@ -65,6 +69,11 @@ class Hyperdrive extends EventEmitter { } */ } + + function onContentReady (err) { + if (err) return self.emit('error', err) + self.emit('content') + } } get version () { @@ -78,34 +87,45 @@ class Hyperdrive extends EventEmitter { _ready (cb) { var self = this - self.metadataFeed.ready(function (err) { + this.metadataFeed.on('error', onerror) + this.metadataFeed.on('append', update) + + this.metadataFeed.ready(err => { if (err) return cb(err) + var keyPair = this.metadataFeed.secretKey ? contentKeyPair(this.metadataFeed.secretKey) : {} + this._contentOpts = contentOptions(this, keyPair.secretKey) + /** + * If a trie is provided as input, ensure that a contentFeed is also provided, then return (this is a checkout). * If the metadata feed is writable: * If the metadata feed has length 0, then the trie should be initialized with the content feed key as metadata. * Else, initialize the trie without metadata and load the content feed key from the header. * If the metadata feed is readable: * Initialize the trie without metadata and load the content feed key from the header. */ - if (self.metadataFeed.writable && !self.metadataFeed.length) { - initialize() + if (this.trie) { + if (!this.contentFeed || !this.metadataFeed) return cb(new Error('Must provide a trie and both content/metadata feeds')) + return done(null) + } else if (this.metadataFeed.writable && !this.metadataFeed.length) { + initialize(keyPair) } else { - restore() + restore(keyPair) } }) - function initialize () { - var keyPair = contentKeyPair(self.metadataFeed.secretKey) - var opts = contentOptions(self, keyPair.secretKey) - self.contentFeed = hypercore(self._storages.content, keyPair.publicKey, opts) + /** + * The first time the hyperdrive is created, we initialize both the trie (metadata feed) and the content feed here. + */ + function initialize (keyPair) { + self.contentFeed = hypercore(self._storages.content, keyPair.publicKey, self._contentOpts) self.contentFeed.on('error', function (err) { self.emit('error', err) }) self.contentFeed.ready(function (err) { if (err) return cb(err) - self.trie = self.trie || hypertrie(null, { + self.trie = hypertrie(null, { feed: self.metadataFeed, metadata: self.contentFeed.key, valueEncoding: messages.Stat @@ -118,35 +138,31 @@ class Hyperdrive extends EventEmitter { }) } - function restore () { + /** + * If the hyperdrive has already been created, wait for the trie (metadata feed) to load. + * If the metadata feed is writable, we can immediately load the content feed from its private key. + * (Otherwise, we need to read the feed's metadata block first) + */ + function restore (keyPair) { self.trie = hypertrie(null, { - feed: self.metadataFeed + feed: self.metadataFeed, + valueEncoding: messages.Stat }) - self.trie.ready(function (err) { - if (err) return cb(err) - - self.trie.getMetadata(function (err, contentKey) { - if (err) return cb(err) - - self.contentFeed = hypercore(self._storages.content, contentKey, contentOptions(self, null)) - self.contentFeed.ready(done) + if (self.metadataFeed.writable) { + self.trie.ready(err => { + if (err) return done(err) + self._ensureContent(done) }) - }) + } else { + self.trie.ready(done) + } } function done (err) { - if (err) return cb(err) - - self.key = self.metadataFeed.key - self.discoveryKey = self.metadataFeed.discoveryKey - - self.metadataFeed.on('update', update) - self.metadataFeed.on('error', onerror) - self.contentFeed.on('error', onerror) - - self.emit('content') - - return cb(null) + if (err) return cb(err) + self.key = self.metadataFeed.key + self.discoveryKey = self.metadataFeed.discoveryKey + return cb(null) } function onerror (err) { @@ -158,6 +174,27 @@ class Hyperdrive extends EventEmitter { } } + _ensureContent (cb) { + this.trie.getMetadata((err, contentKey) => { + if (err) return cb(err) + + this.contentFeed = hypercore(this._storages.content, contentKey, this._contentOpts) + this.contentFeed.ready(err => { + if (err) return cb(err) + + this.contentFeed.on('error', err => this.emit('error', err)) + return cb(null) + }) + }) + } + + _contentReady (cb) { + this.ready(err => { + if (err) return cb(err) + this._ensureContent(cb) + }) + } + createReadStream (name, opts) { if (!opts) opts = {} @@ -168,7 +205,7 @@ class Hyperdrive extends EventEmitter { highWaterMark: opts.highWaterMark || 64 * 1024 }) - this.ready(err => { + this.contentReady(err => { if (err) return stream.destroy(err) this.trie.get(name, (err, st) => { @@ -205,7 +242,7 @@ class Hyperdrive extends EventEmitter { // TODO: support piping through a "split" stream like rabin - this.ready(err => { + this.contentReady(err => { if (err) return proxy.destroy(err) this._lock(_release => { release = _release @@ -300,14 +337,18 @@ class Hyperdrive extends EventEmitter { name = unixify(name) - this._lock(release => { - var st = Stat.directory({ - ...opts, - offset: this.contentFeed.length, - byteOffset: this.contentFeed.byteLength - }) - this.trie.put(name, st, err => { - release(cb, err) + this.ready(err => { + if (err) return cb(err) + + this._lock(release => { + var st = Stat.directory({ + ...opts, + offset: this.contentFeed.length, + byteOffset: this.contentFeed.byteLength + }) + this.trie.put(name, st, err => { + release(cb, err) + }) }) }) } @@ -317,7 +358,7 @@ class Hyperdrive extends EventEmitter { ite.next((err, st) => { if (err) return cb(err) if (name !== '/' && !st) return cb(new errors.FileNotFound(name)) - let st = Stat.directory() + st = Stat.directory() return cb(null, st) }) } @@ -327,9 +368,14 @@ class Hyperdrive extends EventEmitter { if (!opts) opts = {} name = unixify(name) - this.trie.get(name, opts, (err, st) => { - if (err) return this._statDirectory(name, opts, cb) - cb(null, new Stat(st)) + this.ready(err => { + if (err) return cb(err) + + this.trie.get(name, opts, (err, node) => { + if (err) return cb(err) + if (!node) return this._statDirectory(name, opts, cb) + cb(null, new Stat(node.value)) + }) }) } @@ -368,14 +414,16 @@ class Hyperdrive extends EventEmitter { _del (name, cb) { var _release = null + var self = this - this.ready(err => { + this.contentReady(err => { if (err) return cb(err) this._lock(release => { _release = release - this.trie.get(name, (err, st) => { + this.trie.get(name, (err, node) => { if (err) return done(err) - self.contentFeed.clear(st.offset, st.offset + st.blocks, del) + var st = node.value + this.contentFeed.clear(st.offset, st.offset + st.blocks, del) }) }) }) @@ -413,7 +461,7 @@ class Hyperdrive extends EventEmitter { var stream = this.metadataFeed.replicate(opts) - this.ready(err => { + this.contentReady(err => { if (err) return stream.destroy(err) if (stream.destroyed) return this.contentFeed.replicate({ @@ -435,7 +483,26 @@ class Hyperdrive extends EventEmitter { contentFeed: this.contentFeed, trie: versionedTrie, } - return new Hyperdrive(null, null, opts) + return new Hyperdrive(this.storage, this.key, opts) + } + + _closeFile (fd, cb) { + // TODO: implement + process.nextTick(cb, null) + } + + close (fd, cb) { + if (typeof fd === 'number') return this._closeFile(fd, cb || noop) + else cb = fd + if (!cb) cb = noop + + this.contentReady(err => { + if (err) return cb(err) + this.metadataFeed.close(err => { + if (!this.contentFeed) return cb(err) + this.contentFeed.close(cb) + }) + }) } } diff --git a/lib/stat.js b/lib/stat.js index 65e270eb..2d16de94 100644 --- a/lib/stat.js +++ b/lib/stat.js @@ -29,34 +29,36 @@ class Stat { } isSocket () { - this._check(Stat.IFSOCK) + return this._check(Stat.IFSOCK) } isSymbolicLink () { - this._check(Stat.IFLNK) + return this._check(Stat.IFLNK) } isFile () { - this._check(Stat.IFREG) + return this._check(Stat.IFREG) } isBlockDevice () { - this._check(Stat.IFBLK) + return this._check(Stat.IFBLK) } isDirectory () { - this._check(Stat.IFDIR) + return this._check(Stat.IFDIR) } isCharacterDevice () { - this._check(Stat.IFCHR) + return this._check(Stat.IFCHR) } isFIFO () { - this._check(Stat.IFIFO) + return this._check(Stat.IFIFO) } } Stat.file = function (data) { - data.mode = data.mode || DEFAULT_FMODE | Stat.IFREG + data = data || {} + data.mode = (data.mode || DEFAULT_FMODE) | Stat.IFREG return new Stat(data) } Stat.directory = function (data) { - data.mode = data.mode || DEFAULT_DMODE | Stat.IFDIR + data = data || {} + data.mode = (data.mode || DEFAULT_DMODE) | Stat.IFDIR return new Stat(data) } diff --git a/test/basic.js b/test/basic.js index 0f09896b..a2e30246 100644 --- a/test/basic.js +++ b/test/basic.js @@ -50,7 +50,7 @@ tape('write and read (2 parallel)', function (t) { }) }) -tape.skip('write and read (sparse)', function (t) { +tape('write and read (sparse)', function (t) { t.plan(2) var archive = create() @@ -70,7 +70,7 @@ tape.skip('write and read (sparse)', function (t) { }) }) -tape.skip('write and unlink', function (t) { +tape('write and unlink', function (t) { var archive = create() archive.writeFile('/hello.txt', 'world', function (err) { @@ -85,7 +85,7 @@ tape.skip('write and unlink', function (t) { }) }) -tape.skip('root is always there', function (t) { +tape('root is always there', function (t) { var archive = create() archive.access('/', function (err) { @@ -98,7 +98,7 @@ tape.skip('root is always there', function (t) { }) }) -tape.skip('provide keypair', function (t) { +tape('provide keypair', function (t) { var publicKey = new Buffer(sodium.crypto_sign_PUBLICKEYBYTES) var secretKey = new Buffer(sodium.crypto_sign_SECRETKEYBYTES) @@ -109,7 +109,7 @@ tape.skip('provide keypair', function (t) { archive.on('ready', function () { t.ok(archive.writable) t.ok(archive.metadataFeed.writable) - t.ok(archive.contentFeek.writable) + t.ok(archive.contentFeed.writable) t.ok(publicKey.equals(archive.key)) archive.writeFile('/hello.txt', 'world', function (err) { @@ -123,7 +123,7 @@ tape.skip('provide keypair', function (t) { }) }) -tape('download a version', function (t) { +tape.skip('download a version', function (t) { var src = create() src.on('ready', function () { t.ok(src.writable) @@ -163,7 +163,7 @@ tape('download a version', function (t) { } }) -tape.skip('write and read, no cache', function (t) { +tape('write and read, no cache', function (t) { var archive = create({ metadataStorageCacheSize: 0, contentStorageCacheSize: 0, @@ -178,6 +178,7 @@ tape.skip('write and read, no cache', function (t) { t.end() }) }) + var self = this }) tape.skip('closing a read-only, latest clone', function (t) { @@ -193,7 +194,3 @@ tape.skip('closing a read-only, latest clone', function (t) { t.end() }) }) - - - - diff --git a/test/storage.js b/test/storage.js new file mode 100644 index 00000000..7a7ee1b7 --- /dev/null +++ b/test/storage.js @@ -0,0 +1,134 @@ +var tape = require('tape') +var tmp = require('temporary-directory') +var create = require('./helpers/create') +var Hyperdrive = require('..') + +tape('ram storage', function (t) { + var archive = create() + + archive.ready(function () { + t.ok(archive.metadataFeed.writable, 'archive metadata is writable') + t.ok(archive.contentFeed.writable, 'archive content is writable') + t.end() + }) +}) + +tape('dir storage with resume', function (t) { + tmp(function (err, dir, cleanup) { + t.ifError(err) + var archive = new Hyperdrive(dir) + archive.ready(function () { + t.ok(archive.metadataFeed.writable, 'archive metadata is writable') + t.ok(archive.contentFeed.writable, 'archive content is writable') + t.same(archive.version, 0, 'archive has version 0') + archive.close(function (err) { + t.ifError(err) + + var archive2 = new Hyperdrive(dir) + archive2.ready(function (err) { + t.ok(archive2.metadataFeed.writable, 'archive2 metadata is writable') + t.ok(archive2.contentFeed.writable, 'archive2 content is writable') + t.same(archive2.version, 0, 'archive has version 0') + + cleanup(function (err) { + t.ifError(err) + t.end() + }) + }) + }) + }) + }) +}) + +tape('dir storage for non-writable archive', function (t) { + var src = create() + src.ready(function () { + tmp(function (err, dir, cleanup) { + t.ifError(err) + + var clone = new Hyperdrive(dir, src.key) + clone.on('content', function () { + t.ok(!clone.metadataFeed.writable, 'clone metadata not writable') + t.ok(!clone.contentFeed.writable, 'clone content not writable') + t.same(clone.key, src.key, 'keys match') + cleanup(function (err) { + t.ifError(err) + t.end() + }) + }) + + var stream = clone.replicate() + stream.pipe(src.replicate()).pipe(stream) + }) + }) +}) + +tape('dir storage without permissions emits error', function (t) { + t.plan(1) + var archive = new Hyperdrive('/') + archive.on('error', function (err) { + t.ok(err, 'got error') + }) +}) + +tape('write and read (sparse)', function (t) { + t.plan(3) + + tmp(function (err, dir, cleanup) { + t.ifError(err) + var archive = new Hyperdrive(dir) + archive.on('ready', function () { + var clone = create(archive.key, {sparse: true}) + clone.on('ready', function () { + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + var stream = clone.replicate() + stream.pipe(archive.replicate()).pipe(stream) + var readStream = clone.createReadStream('/hello.txt') + readStream.on('error', function (err) { + t.error(err, 'no error') + }) + readStream.on('data', function (data) { + t.same(data.toString(), 'world') + }) + }) + }) + }) + }) +}) + +tape('sparse read/write two files', function (t) { + var archive = create() + archive.on('ready', function () { + var clone = create(archive.key, {sparse: true}) + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + archive.writeFile('/hello2.txt', 'world', function (err) { + t.error(err, 'no error') + var stream = clone.replicate() + stream.pipe(archive.replicate()).pipe(stream) + clone.metadataFeed.update(start) + }) + }) + + function start () { + clone.stat('/hello.txt', function (err, stat) { + t.error(err, 'no error') + t.ok(stat, 'has stat') + clone.readFile('/hello.txt', function (err, data) { + t.error(err, 'no error') + t.same(data.toString(), 'world', 'data ok') + clone.stat('/hello2.txt', function (err, stat) { + t.error(err, 'no error') + t.ok(stat, 'has stat') + clone.readFile('/hello2.txt', function (err, data) { + t.error(err, 'no error') + t.same(data.toString(), 'world', 'data ok') + t.end() + }) + }) + }) + }) + } + }) +}) From 4a5a1bd71a3386f51219cd2793422acc1ade34de Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 30 Jan 2019 13:09:01 -0800 Subject: [PATCH 006/108] Use consts + remove all clear logic temporarily --- index.js | 84 ++++++++++++++++++++++----------------------------- package.json | 3 +- test/basic.js | 38 ++++++++++++----------- 3 files changed, 58 insertions(+), 67 deletions(-) diff --git a/index.js b/index.js index 3d7c769a..a1fd2f25 100644 --- a/index.js +++ b/index.js @@ -55,19 +55,11 @@ class Hyperdrive extends EventEmitter { this.ready(onReady) this.contentReady(onContentReady) - var self = this + const self = this function onReady (err) { if (err) return self.emit('error', err) self.emit('ready') - /* - if (self.latest && !self.metadata.writable) { - self._trackLatest(function (err) { - if (self._closed) return - onerror(err) - }) - } - */ } function onContentReady (err) { @@ -85,7 +77,7 @@ class Hyperdrive extends EventEmitter { } _ready (cb) { - var self = this + const self = this this.metadataFeed.on('error', onerror) this.metadataFeed.on('append', update) @@ -93,7 +85,7 @@ class Hyperdrive extends EventEmitter { this.metadataFeed.ready(err => { if (err) return cb(err) - var keyPair = this.metadataFeed.secretKey ? contentKeyPair(this.metadataFeed.secretKey) : {} + const keyPair = this.metadataFeed.secretKey ? contentKeyPair(this.metadataFeed.secretKey) : {} this._contentOpts = contentOptions(this, keyPair.secretKey) /** @@ -200,7 +192,7 @@ class Hyperdrive extends EventEmitter { name = unixify(name) - var stream = coreByteStream({ + const stream = coreByteStream({ ...opts, highWaterMark: opts.highWaterMark || 64 * 1024 }) @@ -214,8 +206,8 @@ class Hyperdrive extends EventEmitter { st = st.value - var byteOffset = (opts.start) ? st.byteOffset + opts.start : st.byteOffset - var byteLength = (opts.start) ? st.size - opts.start : st.size + let byteOffset = (opts.start) ? st.byteOffset + opts.start : st.byteOffset + let byteLength = (opts.start) ? st.size - opts.start : st.size stream.start({ feed: this.contentFeed, @@ -230,13 +222,25 @@ class Hyperdrive extends EventEmitter { return stream } + createDirectoryStream (name, opts) { + if (!opts) opts = {} + + name = unixify(name) + + const proxy = duplexify() + + this.ready(err => { + if (err) return + }) + } + createWriteStream (name, opts) { if (!opts) opts = {} name = unixify(name) - var self = this - var proxy = duplexify() + const self = this + const proxy = duplexify() var release = null proxy.setReadable(false) @@ -246,11 +250,7 @@ class Hyperdrive extends EventEmitter { if (err) return proxy.destroy(err) this._lock(_release => { release = _release - this.trie.get(name, (err, st) => { - if (err) return proxy.destroy(err) - if (!st) return append(null) - this.contentFeed.clear(st.offset, st.offset + st.blocks, append) - }) + return append() }) }) @@ -261,13 +261,13 @@ class Hyperdrive extends EventEmitter { if (proxy.destroyed) return release() // No one should mutate the content other than us - var byteOffset = self.contentFeed.byteLength - var offset = self.contentFeed.length + let byteOffset = self.contentFeed.byteLength + let offset = self.contentFeed.length self.emit('appending', name, opts) // TODO: revert the content feed if this fails!!!! (add an option to the write stream for this (atomic: true)) - var stream = self.contentFeed.createWriteStream() + const stream = self.contentFeed.createWriteStream() proxy.on('close', done) proxy.on('finish', done) @@ -307,7 +307,7 @@ class Hyperdrive extends EventEmitter { collect(this.createReadStream(name, opts), function (err, bufs) { if (err) return cb(err) - var buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs) + let buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs) cb(null, opts.encoding && opts.encoding !== 'binary' ? buf.toString(opts.encoding) : buf) }) } @@ -321,11 +321,11 @@ class Hyperdrive extends EventEmitter { name = unixify(name) - var bufs = split(buf) // split the input incase it is a big buffer. - var stream = this.createWriteStream(name, opts) + let bufs = split(buf) // split the input incase it is a big buffer. + let stream = this.createWriteStream(name, opts) stream.on('error', cb) stream.on('finish', cb) - for (var i = 0; i < bufs.length; i++) stream.write(bufs[i]) + for (let i = 0; i < bufs.length; i++) stream.write(bufs[i]) stream.end() } @@ -341,7 +341,7 @@ class Hyperdrive extends EventEmitter { if (err) return cb(err) this._lock(release => { - var st = Stat.directory({ + let st = Stat.directory({ ...opts, offset: this.contentFeed.length, byteOffset: this.contentFeed.byteLength @@ -354,7 +354,7 @@ class Hyperdrive extends EventEmitter { } _statDirectory (name, opts, cb) { - var ite = this.trie.iterator(name) + const ite = this.trie.iterator(name) ite.next((err, st) => { if (err) return cb(err) if (name !== '/' && !st) return cb(new errors.FileNotFound(name)) @@ -413,34 +413,22 @@ class Hyperdrive extends EventEmitter { } _del (name, cb) { - var _release = null - var self = this - this.contentReady(err => { if (err) return cb(err) this._lock(release => { - _release = release this.trie.get(name, (err, node) => { if (err) return done(err) - var st = node.value - this.contentFeed.clear(st.offset, st.offset + st.blocks, del) + let st = node.value + this.trie.del(name, err => { + release(cb, err) + }) }) }) }) - - function del (err) { - if (err) return done(err) - self.trie.del(name, done) - } - - function done (err) { - _release(cb, err) - } } unlink (name, cb) { name = unixify(name) - this._del(name, cb || noop) } @@ -459,7 +447,7 @@ class Hyperdrive extends EventEmitter { if (!opts) opts = {} opts.expectedFeeds = 2 - var stream = this.metadataFeed.replicate(opts) + const stream = this.metadataFeed.replicate(opts) this.contentReady(err => { if (err) return stream.destroy(err) @@ -476,7 +464,7 @@ class Hyperdrive extends EventEmitter { } checkout (version, opts) { - var versionedTrie = this.trie.checkout(version) + const versionedTrie = this.trie.checkout(version) opts = { ...opts, metadataFeed: this.metadataFeed, diff --git a/package.json b/package.json index 13e7c52c..0a233318 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,8 @@ "stream-collector": "^1.0.1", "through2": "^3.0.0", "thunky": "^1.0.3", - "unixify": "^1.0.0" + "unixify": "^1.0.0", + "hypercore-byte-stream": "^1.0.0" }, "devDependencies": { "fuzzbuzz": "^1.2.0", diff --git a/test/basic.js b/test/basic.js index a2e30246..e0c05f10 100644 --- a/test/basic.js +++ b/test/basic.js @@ -123,6 +123,26 @@ tape('provide keypair', function (t) { }) }) +tape('write and read, no cache', function (t) { + var archive = create({ + metadataStorageCacheSize: 0, + contentStorageCacheSize: 0, + treeCacheSize: 0 + }) + + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + archive.readFile('/hello.txt', function (err, buf) { + t.error(err, 'no error') + t.same(buf, new Buffer('world')) + t.end() + }) + }) + var self = this +}) + +// TODO: Re-enable the following tests once the `download` and `fetchLatest` APIs are reimplemented. + tape.skip('download a version', function (t) { var src = create() src.on('ready', function () { @@ -163,24 +183,6 @@ tape.skip('download a version', function (t) { } }) -tape('write and read, no cache', function (t) { - var archive = create({ - metadataStorageCacheSize: 0, - contentStorageCacheSize: 0, - treeCacheSize: 0 - }) - - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - archive.readFile('/hello.txt', function (err, buf) { - t.error(err, 'no error') - t.same(buf, new Buffer('world')) - t.end() - }) - }) - var self = this -}) - tape.skip('closing a read-only, latest clone', function (t) { // This is just a sample key of a dead dat var clone = create('1d5e5a628d237787afcbfec7041a16f67ba6895e7aa31500013e94ddc638328d', { From 1859f52eefb098bab3173452048071e8c32136c5 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 31 Jan 2019 05:58:16 -0800 Subject: [PATCH 007/108] Added watch --- index.js | 5 +++++ test/basic.js | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/index.js b/index.js index a1fd2f25..0ec9cc67 100644 --- a/index.js +++ b/index.js @@ -492,6 +492,11 @@ class Hyperdrive extends EventEmitter { }) }) } + + watch (name, onchange) { + name = unixify(name) + return this.trie.watch(name, onchange) + } } module.exports = Hyperdrive diff --git a/test/basic.js b/test/basic.js index e0c05f10..10ed0200 100644 --- a/test/basic.js +++ b/test/basic.js @@ -196,3 +196,30 @@ tape.skip('closing a read-only, latest clone', function (t) { t.end() }) }) + +tape('simple watch', function (t) { + const db = create(null, { valueEncoding: 'utf8' }) + + var watchEvents = 0 + db.ready(err => { + t.error(err, 'no error') + db.watch('/a/path/', () => { + if (++watchEvents === 2) { + t.end() + } + }) + doWrites() + }) + + function doWrites () { + db.writeFile('/a/path/hello', 't1', err => { + t.error(err, 'no error') + db.writeFile('/b/path/hello', 't2', err => { + t.error(err, 'no error') + db.writeFile('/a/path/world', 't3', err => { + t.error(err, 'no error') + }) + }) + }) + } +}) From 48aecac81255634a5e256badf64ceb82ffdf124e Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 31 Jan 2019 08:33:31 -0800 Subject: [PATCH 008/108] Added fuzzing using fuzzbuzz + fixed deprecation warnings --- index.js | 11 ++-- package.json | 3 +- test/fuzzing.js | 169 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 7 deletions(-) create mode 100644 test/fuzzing.js diff --git a/index.js b/index.js index 0ec9cc67..21c7430f 100644 --- a/index.js +++ b/index.js @@ -417,7 +417,8 @@ class Hyperdrive extends EventEmitter { if (err) return cb(err) this._lock(release => { this.trie.get(name, (err, node) => { - if (err) return done(err) + if (err) return release(cb, err) + if (!node) return release(cb, new errors.FileNotFound(name)) let st = node.value this.trie.del(name, err => { release(cb, err) @@ -548,11 +549,11 @@ function contentOptions (self, secretKey) { } function contentKeyPair (secretKey) { - var seed = new Buffer(sodium.crypto_sign_SEEDBYTES) - var context = new Buffer('hyperdri') // 8 byte context + var seed = Buffer.allocUnsafe(sodium.crypto_sign_SEEDBYTES) + var context = Buffer.from('hyperdri', 'utf8') // 8 byte context var keyPair = { - publicKey: new Buffer(sodium.crypto_sign_PUBLICKEYBYTES), - secretKey: new Buffer(sodium.crypto_sign_SECRETKEYBYTES) + publicKey: Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES), + secretKey: Buffer.allocUnsafe(sodium.crypto_sign_SECRETKEYBYTES) } sodium.crypto_kdf_derive_from_key(seed, 1, context, secretKey) diff --git a/package.json b/package.json index 0a233318..13e7c52c 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,7 @@ "stream-collector": "^1.0.1", "through2": "^3.0.0", "thunky": "^1.0.3", - "unixify": "^1.0.0", - "hypercore-byte-stream": "^1.0.0" + "unixify": "^1.0.0" }, "devDependencies": { "fuzzbuzz": "^1.2.0", diff --git a/test/fuzzing.js b/test/fuzzing.js new file mode 100644 index 00000000..e1f56057 --- /dev/null +++ b/test/fuzzing.js @@ -0,0 +1,169 @@ +const tape = require('tape') +const sodium = require('sodium-universal') +const FuzzBuzz = require('fuzzbuzz') + +const create = require('./helpers/create') + +const MAX_PATH_DEPTH = 30 +const MAX_FILE_LENGTH = 1e4 +const CHARACTERS = 1e3 + +class HyperdriveFuzzer extends FuzzBuzz { + constructor (drive, opts) { + super(opts) + this.drive = drive + this.files = new Map() + this.directories = new Map() + + this.validate(this._validate) + this.setup(this._setup) + + this.add(10, this.writeFile) + this.add(5, this.deleteFile) + this.add(3, this.statFile) + this.add(2, this.deleteInvalidFile) + } + + _select (map) { + let idx = this.randomInt(map.size -1) + if (idx < 0) return null + + let ite = map.entries() + while (idx--) ite.next() + return ite.next().value + } + _selectFile () { + return this._select(this.files) + } + _selectDirectory () { + return this._select(this.directories) + } + + _fileName () { + let depth = this.randomInt(MAX_PATH_DEPTH) + let name = (new Array(depth)).fill(0).map(() => String.fromCharCode(this.randomInt(CHARACTERS))).join('/') + return name + } + _createFile () { + let name = this._fileName() + let content = Buffer.allocUnsafe(this.randomInt(MAX_FILE_LENGTH)).fill(this.randomInt(10)) + return { name, content } + } + _deleteFile (name) { + return new Promise((resolve, reject) => { + this.drive.unlink(name, err => { + if (err) return reject(err) + this.files.delete(name) + return resolve() + }) + }) + } + + // START Public operations + + // File-level operations + + async writeFile () { + let { name, content } = this._createFile() + return new Promise((resolve, reject) => { + this.drive.writeFile(name, content, err => { + if (err) return reject(err) + this.files.set(name, content) + return resolve() + }) + }) + } + + async deleteFile () { + let selected = this._selectFile() + if (!selected) return + + let fileName = selected[0] + return this._deleteFile(fileName) + } + + async deleteInvalidFile () { + let name = this._fileName() + while (this.files.get(name)) name = this._fileName() + try { + await this._deleteFile(name) + } catch (err) { + if (err && err.code !== 'ENOENT') throw err + } + } + + statFile () { + let selected = this._selectFile() + if (!selected) return + + let [fileName, contents] = selected + return new Promise((resolve, reject) => { + this.drive.stat(fileName, (err, st) => { + if (err) return reject(err) + if (!st) return reject(new Error(`File ${fileName} should exist does not exist.`)) + if (st.size !== contents.length) return reject(new Error(`Incorrect content length for file ${fileName}.`)) + return resolve() + }) + }) + } + + // END Public operations + + _validateFile (name, content) { + return new Promise((resolve, reject) => { + this.drive.readFile(name, (err, data) => { + if (err) return reject(err) + if (!data.equals(content)) return reject(new Error(`Read data for ${name} does not match written content.`)) + return resolve() + }) + }) + } + _validateDirectory (name, list) { + return new Promise((resolve, reject) => { + this.drive.readdir(name, (err, list) => { + if (err) return reject(err) + let fileSet = new Set(list) + for (const file of list) { + if (!fileSet.has(file)) return reject(new Error(`Directory does not contain expected file: ${file}`)) + fileSet.delete(file) + } + if (fileSet.size) return reject(new Error(`Directory contains unexpected files: ${fileSet}`)) + return resolve() + }) + }) + } + + async _validate () { + for (const [fileName, content] of this.files) { + await this._validateFile(fileName, content) + } + for (const [dirName, list] of this.directories) { + await this._validateDirectory(dirName, list) + } + } + + _setup () { + return new Promise((resolve, reject) => { + this.drive.ready(err => { + if (err) return reject(err) + return resolve() + }) + }) + } +} + +tape.only('100000 mixed operations', async t => { + t.plan(1) + + let drive = create() + const fuzz = new HyperdriveFuzzer(drive, { + seed: 'hyperdrive' + }) + + try { + await fuzz.run(100000) + t.pass('fuzzing succeeded') + } catch (err) { + t.error(err, 'no error') + } +}) From 27ae10a3af3dfa37904b86cdbe7d977bd2a17355 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 1 Feb 2019 07:39:03 -0800 Subject: [PATCH 009/108] Added createDirectoryStream + updated version handling + fixed deprecation warnings --- index.js | 32 +++++++++--- test/basic.js | 106 +++++++++++++++++++++++++++++++++++--- test/fuzzing.js | 132 +++++++++++++++++++++++++++--------------------- test/storage.js | 4 +- 4 files changed, 201 insertions(+), 73 deletions(-) diff --git a/index.js b/index.js index 21c7430f..e4ffb664 100644 --- a/index.js +++ b/index.js @@ -8,6 +8,8 @@ const raf = require('random-access-file') const mutexify = require('mutexify') const duplexify = require('duplexify') const sodium = require('sodium-universal') +const through = require('through2') +const pump = require('pump') const hypercore = require('hypercore') const hypertrie = require('hypertrie') @@ -34,7 +36,7 @@ class Hyperdrive extends EventEmitter { this._storages = defaultStorage(this, storage, opts) - this.metadataFeed = opts.metatadataFeed || hypercore(this._storages.metadata, key, { + this.metadataFeed = opts.metadataFeed || hypercore(this._storages.metadata, key, { secretKey: opts.secretKey, sparse: opts.sparseMetadata, createIfMissing: opts.createIfMissing, @@ -45,7 +47,6 @@ class Hyperdrive extends EventEmitter { this.contentFeed = opts.contentFeed || null this.storage = storage - this._checkout = opts._checkout this._contentOpts = null this._lock = mutexify() @@ -69,7 +70,8 @@ class Hyperdrive extends EventEmitter { } get version () { - return this._checkout ? this.trie.version : (this.metadataFeed.length ? this.metadataFeed.length - 1 : 0) + // TODO: The trie version starts at 1, so the empty hyperdrive version is also 1. This should be 0. + return this.trie.version } get writable () { @@ -183,6 +185,7 @@ class Hyperdrive extends EventEmitter { _contentReady (cb) { this.ready(err => { if (err) return cb(err) + if (this.contentFeed) return cb(null) this._ensureContent(cb) }) } @@ -227,11 +230,24 @@ class Hyperdrive extends EventEmitter { name = unixify(name) - const proxy = duplexify() + const proxy = duplexify.obj() + proxy.setWritable(false) this.ready(err => { if (err) return + let stream = pump( + this.trie.createReadStream(name, opts), + through.obj((chunk, enc, cb) => { + return cb(null, { + path: chunk.key, + stat: chunk.value + }) + }) + ) + proxy.setReadable(stream) }) + + return proxy } createWriteStream (name, opts) { @@ -316,7 +332,7 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'function') return this.writeFile(name, buf, null, opts) if (typeof opts === 'string') opts = {encoding: opts} if (!opts) opts = {} - if (typeof buf === 'string') buf = new Buffer(buf, opts.encoding || 'utf-8') + if (typeof buf === 'string') buf = Buffer.from(buf, opts.encoding || 'utf-8') if (!cb) cb = noop name = unixify(name) @@ -409,7 +425,11 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'function') return this.readdir(name, null, opts) name = unixify(name) - return this.trie.list(name, opts, cb) + let dirStream = this.createDirectoryStream(name, opts) + collect(dirStream, (err, stats) => { + if (err) return cb(err) + return cb(null, stats.map(s => s.path)) + }) } _del (name, cb) { diff --git a/test/basic.js b/test/basic.js index 10ed0200..30ba388a 100644 --- a/test/basic.js +++ b/test/basic.js @@ -37,7 +37,7 @@ tape('write and read (2 parallel)', function (t) { t.error(err, 'no error') archive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') - t.same(buf, new Buffer('world')) + t.same(buf, Buffer.from('world')) }) }) @@ -45,7 +45,7 @@ tape('write and read (2 parallel)', function (t) { t.error(err, 'no error') archive.readFile('/world.txt', function (err, buf) { t.error(err, 'no error') - t.same(buf, new Buffer('hello')) + t.same(buf, Buffer.from('hello')) }) }) }) @@ -99,8 +99,8 @@ tape('root is always there', function (t) { }) tape('provide keypair', function (t) { - var publicKey = new Buffer(sodium.crypto_sign_PUBLICKEYBYTES) - var secretKey = new Buffer(sodium.crypto_sign_SECRETKEYBYTES) + var publicKey = Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES) + var secretKey = Buffer.allocUnsafe(sodium.crypto_sign_SECRETKEYBYTES) sodium.crypto_sign_keypair(publicKey, secretKey) @@ -116,7 +116,7 @@ tape('provide keypair', function (t) { t.error(err, 'no error') archive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') - t.same(buf, new Buffer('world')) + t.same(buf, Buffer.from('world')) t.end() }) }) @@ -134,7 +134,7 @@ tape('write and read, no cache', function (t) { t.error(err, 'no error') archive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') - t.same(buf, new Buffer('world')) + t.same(buf, Buffer.from('world')) t.end() }) }) @@ -198,7 +198,7 @@ tape.skip('closing a read-only, latest clone', function (t) { }) tape('simple watch', function (t) { - const db = create(null, { valueEncoding: 'utf8' }) + const db = create(null) var watchEvents = 0 db.ready(err => { @@ -223,3 +223,95 @@ tape('simple watch', function (t) { }) } }) + +tape('simple checkout', function (t) { + const drive = create(null) + + drive.writeFile('/hello', 'world', err => { + t.error(err, 'no error') + let version = drive.version + drive.readFile('/hello', (err, data) => { + t.error(err, 'no error') + t.same(data, Buffer.from('world')) + drive.unlink('/hello', err => { + t.error(err, 'no error') + drive.readFile('/hello', (err, data) => { + t.true(err) + t.same(err.code, 'ENOENT') + testCheckout(version) + }) + }) + }) + }) + + function testCheckout (version) { + let oldVersion = drive.checkout(version) + oldVersion.readFile('/hello', (err, data) => { + t.error(err, 'no error') + t.same(data, Buffer.from('world')) + t.end() + }) + } +}) + +tape('can read a single directory', async function (t) { + const drive = create(null) + + let files = ['a', 'b', 'c', 'd', 'e', 'f'] + let fileSet = new Set(files) + + for (let file of files) { + await insertFile(file, 'a small file') + } + + drive.readdir('/', (err, files) => { + t.error(err, 'no error') + for (let file of files) { + t.true(fileSet.has(file), 'correct file was listed') + fileSet.delete(file) + } + t.same(fileSet.size, 0, 'all files were listed') + t.end() + }) + + function insertFile (name, content) { + return new Promise((resolve, reject) => { + drive.writeFile(name, content, err => { + if (err) return reject(err) + return resolve() + }) + }) + } +}) + +tape('can stream a large directory', async function (t) { + const drive = create(null) + + let files = new Array(1000).fill(0).map((_, idx) => '' + idx) + let fileSet = new Set(files) + + for (let file of files) { + await insertFile(file, 'a small file') + } + + let stream = drive.createDirectoryStream('/') + stream.on('data', ({ path, stat }) => { + if (!fileSet.has(path)) { + return t.fail('an incorrect file was streamed') + } + fileSet.delete(path) + }) + stream.on('end', () => { + t.same(fileSet.size, 0, 'all files were streamed') + t.end() + }) + + function insertFile (name, content) { + return new Promise((resolve, reject) => { + drive.writeFile(name, content, err => { + if (err) return reject(err) + return resolve() + }) + }) + } +}) diff --git a/test/fuzzing.js b/test/fuzzing.js index e1f56057..3d6f7dc1 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -1,22 +1,16 @@ const tape = require('tape') const sodium = require('sodium-universal') -const FuzzBuzz = require('fuzzbuzz') - +const FuzzBuzz = require('fuzzbuzz') const create = require('./helpers/create') const MAX_PATH_DEPTH = 30 const MAX_FILE_LENGTH = 1e4 const CHARACTERS = 1e3 +const INVALID_CHARS = new Set(['/', '\\', '?', '%', '*', ':', '|', '"', '<', '>', '.', ' ']) class HyperdriveFuzzer extends FuzzBuzz { - constructor (drive, opts) { + constructor (opts) { super(opts) - this.drive = drive - this.files = new Map() - this.directories = new Map() - - this.validate(this._validate) - this.setup(this._setup) this.add(10, this.writeFile) this.add(5, this.deleteFile) @@ -24,6 +18,8 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(2, this.deleteInvalidFile) } + // START Helper functions. + _select (map) { let idx = this.randomInt(map.size -1) if (idx < 0) return null @@ -39,9 +35,15 @@ class HyperdriveFuzzer extends FuzzBuzz { return this._select(this.directories) } + _validChar () { + do { + var char = String.fromCharCode(this.randomInt(CHARACTERS)) + } while(INVALID_CHARS.has(char)) + return char + } _fileName () { let depth = this.randomInt(MAX_PATH_DEPTH) - let name = (new Array(depth)).fill(0).map(() => String.fromCharCode(this.randomInt(CHARACTERS))).join('/') + let name = (new Array(depth)).fill(0).map(() => this._validChar()).join('/') return name } _createFile () { @@ -54,61 +56,27 @@ class HyperdriveFuzzer extends FuzzBuzz { this.drive.unlink(name, err => { if (err) return reject(err) this.files.delete(name) - return resolve() + return resolve({ type: 'delete', name }) }) }) } - // START Public operations + // START FuzzBuzz interface - // File-level operations + _setup () { + this.drive = create() + this.files = new Map() + this.directories = new Map() + this.log = [] - async writeFile () { - let { name, content } = this._createFile() return new Promise((resolve, reject) => { - this.drive.writeFile(name, content, err => { + this.drive.ready(err => { if (err) return reject(err) - this.files.set(name, content) return resolve() }) }) } - async deleteFile () { - let selected = this._selectFile() - if (!selected) return - - let fileName = selected[0] - return this._deleteFile(fileName) - } - - async deleteInvalidFile () { - let name = this._fileName() - while (this.files.get(name)) name = this._fileName() - try { - await this._deleteFile(name) - } catch (err) { - if (err && err.code !== 'ENOENT') throw err - } - } - - statFile () { - let selected = this._selectFile() - if (!selected) return - - let [fileName, contents] = selected - return new Promise((resolve, reject) => { - this.drive.stat(fileName, (err, st) => { - if (err) return reject(err) - if (!st) return reject(new Error(`File ${fileName} should exist does not exist.`)) - if (st.size !== contents.length) return reject(new Error(`Incorrect content length for file ${fileName}.`)) - return resolve() - }) - }) - } - - // END Public operations - _validateFile (name, content) { return new Promise((resolve, reject) => { this.drive.readFile(name, (err, data) => { @@ -142,25 +110,73 @@ class HyperdriveFuzzer extends FuzzBuzz { } } - _setup () { + + async call (ops) { + let res = await super.call(ops) + this.log.push(res) + } + + // START Fuzzing operations + + async writeFile () { + let { name, content } = this._createFile() return new Promise((resolve, reject) => { - this.drive.ready(err => { + this.drive.writeFile(name, content, err => { if (err) return reject(err) - return resolve() + this.files.set(name, content) + return resolve({ type: 'write', name, content }) + }) + }) + } + + async deleteFile () { + let selected = this._selectFile() + if (!selected) return + + let fileName = selected[0] + return this._deleteFile(fileName) + } + + async deleteInvalidFile () { + let name = this._fileName() + while (this.files.get(name)) name = this._fileName() + try { + await this._deleteFile(name) + } catch (err) { + if (err && err.code !== 'ENOENT') throw err + } + } + + statFile () { + let selected = this._selectFile() + if (!selected) return + + let [fileName, contents] = selected + return new Promise((resolve, reject) => { + this.drive.stat(fileName, (err, st) => { + if (err) return reject(err) + if (!st) return reject(new Error(`File ${fileName} should exist does not exist.`)) + if (st.size !== contents.length) return reject(new Error(`Incorrect content length for file ${fileName}.`)) + return resolve({ type: 'stat', fileName, stat: st }) }) }) } } -tape.only('100000 mixed operations', async t => { +module.exports = HyperdriveFuzzer + +tape('100000 mixed operations', async t => { t.plan(1) - let drive = create() - const fuzz = new HyperdriveFuzzer(drive, { + const fuzz = new HyperdriveFuzzer({ seed: 'hyperdrive' }) try { + /* + let failingIteration = await fuzz.bisect(100000) + t.true(failingIteration, `failed on iteration ${failingIteration}`) + */ await fuzz.run(100000) t.pass('fuzzing succeeded') } catch (err) { diff --git a/test/storage.js b/test/storage.js index 7a7ee1b7..05d5598e 100644 --- a/test/storage.js +++ b/test/storage.js @@ -20,7 +20,7 @@ tape('dir storage with resume', function (t) { archive.ready(function () { t.ok(archive.metadataFeed.writable, 'archive metadata is writable') t.ok(archive.contentFeed.writable, 'archive content is writable') - t.same(archive.version, 0, 'archive has version 0') + t.same(archive.version, 1, 'archive has version 1') archive.close(function (err) { t.ifError(err) @@ -28,7 +28,7 @@ tape('dir storage with resume', function (t) { archive2.ready(function (err) { t.ok(archive2.metadataFeed.writable, 'archive2 metadata is writable') t.ok(archive2.contentFeed.writable, 'archive2 content is writable') - t.same(archive2.version, 0, 'archive has version 0') + t.same(archive2.version, 1, 'archive has version 1') cleanup(function (err) { t.ifError(err) From 2d90fdfa4fa009122c56ec2a93cd1b761bddc4b7 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 4 Feb 2019 03:15:36 -0800 Subject: [PATCH 010/108] Added _contentFeedLength so mkdir needn't lock. this.trie -> this._db --- index.js | 97 ++++++++++++++++++++++++------------------------- test/fuzzing.js | 34 ++++++++++++++++- 2 files changed, 80 insertions(+), 51 deletions(-) diff --git a/index.js b/index.js index e4ffb664..593317bf 100644 --- a/index.js +++ b/index.js @@ -43,11 +43,13 @@ class Hyperdrive extends EventEmitter { storageCacheSize: opts.metadataStorageCacheSize, valueEncoding: 'binary' }) - this.trie = opts.trie + this._db = opts.db this.contentFeed = opts.contentFeed || null this.storage = storage this._contentOpts = null + this._contentFeedLength = null + this._contentFeedByteLength = null this._lock = mutexify() this.ready = thunky(this._ready.bind(this)) @@ -71,7 +73,7 @@ class Hyperdrive extends EventEmitter { get version () { // TODO: The trie version starts at 1, so the empty hyperdrive version is also 1. This should be 0. - return this.trie.version + return this._db.version } get writable () { @@ -91,15 +93,15 @@ class Hyperdrive extends EventEmitter { this._contentOpts = contentOptions(this, keyPair.secretKey) /** - * If a trie is provided as input, ensure that a contentFeed is also provided, then return (this is a checkout). + * If a db is provided as input, ensure that a contentFeed is also provided, then return (this is a checkout). * If the metadata feed is writable: - * If the metadata feed has length 0, then the trie should be initialized with the content feed key as metadata. - * Else, initialize the trie without metadata and load the content feed key from the header. + * If the metadata feed has length 0, then the db should be initialized with the content feed key as metadata. + * Else, initialize the db without metadata and load the content feed key from the header. * If the metadata feed is readable: - * Initialize the trie without metadata and load the content feed key from the header. + * Initialize the db without metadata and load the content feed key from the header. */ - if (this.trie) { - if (!this.contentFeed || !this.metadataFeed) return cb(new Error('Must provide a trie and both content/metadata feeds')) + if (this._db) { + if (!this.contentFeed || !this.metadataFeed) return cb(new Error('Must provide a db and both content/metadata feeds')) return done(null) } else if (this.metadataFeed.writable && !this.metadataFeed.length) { initialize(keyPair) @@ -109,7 +111,7 @@ class Hyperdrive extends EventEmitter { }) /** - * The first time the hyperdrive is created, we initialize both the trie (metadata feed) and the content feed here. + * The first time the hyperdrive is created, we initialize both the db (metadata feed) and the content feed here. */ function initialize (keyPair) { self.contentFeed = hypercore(self._storages.content, keyPair.publicKey, self._contentOpts) @@ -119,13 +121,13 @@ class Hyperdrive extends EventEmitter { self.contentFeed.ready(function (err) { if (err) return cb(err) - self.trie = hypertrie(null, { + self._db = hypertrie(null, { feed: self.metadataFeed, metadata: self.contentFeed.key, valueEncoding: messages.Stat }) - self.trie.ready(function (err) { + self._db.ready(function (err) { if (err) return cb(err) return done(null) }) @@ -133,22 +135,22 @@ class Hyperdrive extends EventEmitter { } /** - * If the hyperdrive has already been created, wait for the trie (metadata feed) to load. + * If the hyperdrive has already been created, wait for the db (metadata feed) to load. * If the metadata feed is writable, we can immediately load the content feed from its private key. * (Otherwise, we need to read the feed's metadata block first) */ function restore (keyPair) { - self.trie = hypertrie(null, { + self._db = hypertrie(null, { feed: self.metadataFeed, valueEncoding: messages.Stat }) if (self.metadataFeed.writable) { - self.trie.ready(err => { + self._db.ready(err => { if (err) return done(err) self._ensureContent(done) }) } else { - self.trie.ready(done) + self._db.ready(done) } } @@ -169,13 +171,16 @@ class Hyperdrive extends EventEmitter { } _ensureContent (cb) { - this.trie.getMetadata((err, contentKey) => { + this._db.getMetadata((err, contentKey) => { if (err) return cb(err) this.contentFeed = hypercore(this._storages.content, contentKey, this._contentOpts) this.contentFeed.ready(err => { if (err) return cb(err) + this._contentFeedByteLength = this.contentFeed.byteLength + this._contentFeedLength = this.contentFeed.length + this.contentFeed.on('error', err => this.emit('error', err)) return cb(null) }) @@ -203,7 +208,7 @@ class Hyperdrive extends EventEmitter { this.contentReady(err => { if (err) return stream.destroy(err) - this.trie.get(name, (err, st) => { + this._db.get(name, (err, st) => { if (err) return stream.destroy(err) if (!st) return stream.destroy(new errors.FileNotFound(name)) @@ -236,11 +241,11 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return let stream = pump( - this.trie.createReadStream(name, opts), + this._db.createReadStream(name, opts), through.obj((chunk, enc, cb) => { return cb(null, { path: chunk.key, - stat: chunk.value + stat: new Stat(chunk.value) }) }) ) @@ -299,7 +304,7 @@ class Hyperdrive extends EventEmitter { }) proxy.cork() - self.trie.put(name, st, function (err) { + self._db.put(name, st, function (err) { if (err) return proxy.destroy(err) self.emit('append', name, opts) proxy.uncork() @@ -310,6 +315,8 @@ class Hyperdrive extends EventEmitter { function done () { proxy.removeListener('close', done) proxy.removeListener('finish', done) + self._contentFeedLength = self.contentFeed.length + self._contentFeedByteLength = self.contentFeed.byteLength release() } } @@ -355,22 +362,17 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return cb(err) - - this._lock(release => { - let st = Stat.directory({ - ...opts, - offset: this.contentFeed.length, - byteOffset: this.contentFeed.byteLength - }) - this.trie.put(name, st, err => { - release(cb, err) - }) + let st = Stat.directory({ + ...opts, + offset: this._contentFeedLength, + byteOffset: this._contentFeedByteLength }) + this._db.put(name, st, cb) }) } _statDirectory (name, opts, cb) { - const ite = this.trie.iterator(name) + const ite = this._db.iterator(name) ite.next((err, st) => { if (err) return cb(err) if (name !== '/' && !st) return cb(new errors.FileNotFound(name)) @@ -387,7 +389,7 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return cb(err) - this.trie.get(name, opts, (err, node) => { + this._db.get(name, opts, (err, node) => { if (err) return cb(err) if (!node) return this._statDirectory(name, opts, cb) cb(null, new Stat(node.value)) @@ -426,24 +428,20 @@ class Hyperdrive extends EventEmitter { name = unixify(name) let dirStream = this.createDirectoryStream(name, opts) - collect(dirStream, (err, stats) => { + this._db.list(name, (err, list) => { if (err) return cb(err) - return cb(null, stats.map(s => s.path)) + return cb(null, list.map(st => name === '/' ? st.key : path.basename(name, st.key))) }) } _del (name, cb) { - this.contentReady(err => { + this.ready(err => { if (err) return cb(err) - this._lock(release => { - this.trie.get(name, (err, node) => { - if (err) return release(cb, err) - if (!node) return release(cb, new errors.FileNotFound(name)) - let st = node.value - this.trie.del(name, err => { - release(cb, err) - }) - }) + this._db.del(name, (err, node) => { + if (err) return cb(err) + if (!node) return cb(new errors.FileNotFound(name)) + // TODO: Need to check if it's a directory, and the directory was not found + return cb(null) }) }) } @@ -457,9 +455,10 @@ class Hyperdrive extends EventEmitter { if (!cb) cb = noop name = unixify(name) - this.readdir(name, function (err, list) { + let stream = this._db.iterator(name) + stream.next((err, val) => { if (err) return cb(err) - if (list.length) return cb(new errors.DirectoryNotEmpty(name)) + if (val) return cb(new errors.DirectoryNotEmpty(name)) self._del(name, cb) }) } @@ -485,12 +484,12 @@ class Hyperdrive extends EventEmitter { } checkout (version, opts) { - const versionedTrie = this.trie.checkout(version) + const versionedTrie = this._db.checkout(version) opts = { ...opts, metadataFeed: this.metadataFeed, contentFeed: this.contentFeed, - trie: versionedTrie, + db: versionedTrie, } return new Hyperdrive(this.storage, this.key, opts) } @@ -516,7 +515,7 @@ class Hyperdrive extends EventEmitter { watch (name, onchange) { name = unixify(name) - return this.trie.watch(name, onchange) + return this._db.watch(name, onchange) } } diff --git a/test/fuzzing.js b/test/fuzzing.js index 3d6f7dc1..d6bb6821 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -8,12 +8,20 @@ const MAX_FILE_LENGTH = 1e4 const CHARACTERS = 1e3 const INVALID_CHARS = new Set(['/', '\\', '?', '%', '*', ':', '|', '"', '<', '>', '.', ' ']) +class Peer { + constructor (drive, files) { + this.drive = drive + this.files = files + } +} + class HyperdriveFuzzer extends FuzzBuzz { constructor (opts) { super(opts) this.add(10, this.writeFile) this.add(5, this.deleteFile) + // this.add(5, this.existingFileOverwrite) this.add(3, this.statFile) this.add(2, this.deleteInvalidFile) } @@ -67,6 +75,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.drive = create() this.files = new Map() this.directories = new Map() + this.streams = new Map() this.log = [] return new Promise((resolve, reject) => { @@ -151,16 +160,37 @@ class HyperdriveFuzzer extends FuzzBuzz { let selected = this._selectFile() if (!selected) return - let [fileName, contents] = selected + let [fileName, content] = selected return new Promise((resolve, reject) => { this.drive.stat(fileName, (err, st) => { if (err) return reject(err) if (!st) return reject(new Error(`File ${fileName} should exist does not exist.`)) - if (st.size !== contents.length) return reject(new Error(`Incorrect content length for file ${fileName}.`)) + if (st.size !== content.length) return reject(new Error(`Incorrect content length for file ${fileName}.`)) return resolve({ type: 'stat', fileName, stat: st }) }) }) } + + existingFileOverwrite () { + let selected = this._selectFile() + if (!selected) return + let [fileName, content] = selected + + let { content: newContent } = this._createFile() + + return new Promise((resolve, reject) => { + let writeStream = this.drive.createWriteStream(fileName) + writeStream.on('error', err => reject(err)) + writeStream.on('finish', () => { + this.files.set(fileName, newContent) + resolve() + }) + writeStream.write(newContent) + }) + } + + readFileStream () { + } } module.exports = HyperdriveFuzzer From c532a2eea2d944dd6049f001ced0a8cfb7d3d27e Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 4 Feb 2019 03:19:44 -0800 Subject: [PATCH 011/108] Added existingFileOverwrite to fuzz tests --- test/fuzzing.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/fuzzing.js b/test/fuzzing.js index d6bb6821..1ab93559 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -21,7 +21,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(10, this.writeFile) this.add(5, this.deleteFile) - // this.add(5, this.existingFileOverwrite) + this.add(5, this.existingFileOverwrite) this.add(3, this.statFile) this.add(2, this.deleteInvalidFile) } @@ -185,7 +185,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.files.set(fileName, newContent) resolve() }) - writeStream.write(newContent) + writeStream.end(newContent) }) } @@ -203,10 +203,6 @@ tape('100000 mixed operations', async t => { }) try { - /* - let failingIteration = await fuzz.bisect(100000) - t.true(failingIteration, `failed on iteration ${failingIteration}`) - */ await fuzz.run(100000) t.pass('fuzzing succeeded') } catch (err) { From 1689498b3e6bdf5f033971860c82a5bef26964d3 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 4 Feb 2019 04:40:15 -0800 Subject: [PATCH 012/108] Added randomReadStream to fuzzer --- test/fuzzing.js | 83 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/test/fuzzing.js b/test/fuzzing.js index 1ab93559..522afb3e 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -1,20 +1,15 @@ const tape = require('tape') const sodium = require('sodium-universal') +const collect = require('stream-collector') + const FuzzBuzz = require('fuzzbuzz') const create = require('./helpers/create') const MAX_PATH_DEPTH = 30 -const MAX_FILE_LENGTH = 1e4 +const MAX_FILE_LENGTH = 1e3 const CHARACTERS = 1e3 const INVALID_CHARS = new Set(['/', '\\', '?', '%', '*', ':', '|', '"', '<', '>', '.', ' ']) -class Peer { - constructor (drive, files) { - this.drive = drive - this.files = files - } -} - class HyperdriveFuzzer extends FuzzBuzz { constructor (opts) { super(opts) @@ -24,6 +19,8 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(5, this.existingFileOverwrite) this.add(3, this.statFile) this.add(2, this.deleteInvalidFile) + this.add(2, this.randomReadStream) + this.add(1, this.writeAndMkdir) } // START Helper functions. @@ -56,7 +53,7 @@ class HyperdriveFuzzer extends FuzzBuzz { } _createFile () { let name = this._fileName() - let content = Buffer.allocUnsafe(this.randomInt(MAX_FILE_LENGTH)).fill(this.randomInt(10)) + let content = Buffer.allocUnsafe(this.randomInt(MAX_FILE_LENGTH)).fill(0).map(() => this.randomInt(10)) return { name, content } } _deleteFile (name) { @@ -86,9 +83,13 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } + _validationDrive () { + return this.drive + } _validateFile (name, content) { + let drive = this._validationDrive() return new Promise((resolve, reject) => { - this.drive.readFile(name, (err, data) => { + drive.readFile(name, (err, data) => { if (err) return reject(err) if (!data.equals(content)) return reject(new Error(`Read data for ${name} does not match written content.`)) return resolve() @@ -96,8 +97,9 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } _validateDirectory (name, list) { + let drive = this._validationDrive() return new Promise((resolve, reject) => { - this.drive.readdir(name, (err, list) => { + drive.readdir(name, (err, list) => { if (err) return reject(err) let fileSet = new Set(list) for (const file of list) { @@ -109,7 +111,6 @@ class HyperdriveFuzzer extends FuzzBuzz { }) }) } - async _validate () { for (const [fileName, content] of this.files) { await this._validateFile(fileName, content) @@ -189,13 +190,52 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } - readFileStream () { + writeAndMkdir () { + } + + randomReadStream () { + let selected = this._selectFile() + if (!selected) return + let [fileName, content] = selected + + return new Promise((resolve, reject) => { + let drive = this._validationDrive() + let start = this.randomInt(content.length) + let stream = drive.createReadStream(fileName, { + start + }) + collect(stream, (err, bufs) => { + if (err) return reject(err) + let buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs) + if (!buf.equals(content.slice(start))) return reject(new Error('Read stream does not match content slice.')) + return resolve() + }) + }) + } +} + +class SparseHyperdriveFuzzer extends HyperdriveFuzzer { + constructor(opts) { + super(opts) + } + _setup () { + return super._setup().then(() => { + this.remoteDrive = create(this.drive.key) + this.remoteDrive.ready(err => { + if (err) throw err + let s1 = this.remoteDrive.replicate({ live: true }) + s1.pipe(this.drive.replicate({ live: true })).pipe(s1) + }) + }) + } + _validationDrive () { + return this.remoteDrive } } module.exports = HyperdriveFuzzer -tape('100000 mixed operations', async t => { +tape('100000 mixed operations, single drive', async t => { t.plan(1) const fuzz = new HyperdriveFuzzer({ @@ -209,3 +249,18 @@ tape('100000 mixed operations', async t => { t.error(err, 'no error') } }) + +tape.only('10000 mixed operations, replicating drives', async t => { + t.plan(1) + + const fuzz = new SparseHyperdriveFuzzer({ + seed: 'hyperdrive2' + }) + + try { + await fuzz.run(10000) + t.pass('fuzzing succeeded') + } catch (err) { + t.error(err, 'no error') + } +}) From 7e8a103985c6a014129bca28478abbea921622c4 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 5 Feb 2019 03:23:58 -0800 Subject: [PATCH 013/108] Update mkdir to use condition function --- index.js | 9 ++++- lib/errors.js | 11 ++++++- test/fuzzing.js | 87 ++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 93 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index 593317bf..5020f41f 100644 --- a/index.js +++ b/index.js @@ -367,8 +367,15 @@ class Hyperdrive extends EventEmitter { offset: this._contentFeedLength, byteOffset: this._contentFeedByteLength }) - this._db.put(name, st, cb) + this._db.put(name, st, { + condition: ifNotExists + }, cb) }) + + function ifNotExists (oldNode, newNode, cb) { + if (oldNode) return cb(new errors.PathAlreadyExists(name)) + return cb(null, true) + } } _statDirectory (name, opts, cb) { diff --git a/lib/errors.js b/lib/errors.js index 108325b8..40a57236 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -16,7 +16,16 @@ class DirectoryNotEmpty extends CustomError { } } +class PathAlreadyExists extends CustomError { + constructor (dirName) { + super(`Path ${dirName} already exists.`) + this.code = 'EEXIST' + this.errno = 17 + } +} + module.exports = { FileNotFound, - DirectoryNotEmpty + DirectoryNotEmpty, + PathAlreadyExists } diff --git a/test/fuzzing.js b/test/fuzzing.js index 522afb3e..45de6b29 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -18,6 +18,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(5, this.deleteFile) this.add(5, this.existingFileOverwrite) this.add(3, this.statFile) + this.add(3, this.statDirectory) this.add(2, this.deleteInvalidFile) this.add(2, this.randomReadStream) this.add(1, this.writeAndMkdir) @@ -43,12 +44,14 @@ class HyperdriveFuzzer extends FuzzBuzz { _validChar () { do { var char = String.fromCharCode(this.randomInt(CHARACTERS)) - } while(INVALID_CHARS.has(char)) + } while (INVALID_CHARS.has(char)) return char } _fileName () { - let depth = this.randomInt(MAX_PATH_DEPTH) - let name = (new Array(depth)).fill(0).map(() => this._validChar()).join('/') + let depth = Math.max(this.randomInt(MAX_PATH_DEPTH), 1) + do { + var name = (new Array(depth)).fill(0).map(() => this._validChar()).join('/') + } while (this.files.get(name) || this.directories.get(name)) return name } _createFile () { @@ -97,6 +100,7 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } _validateDirectory (name, list) { + /* let drive = this._validationDrive() return new Promise((resolve, reject) => { drive.readdir(name, (err, list) => { @@ -110,6 +114,7 @@ class HyperdriveFuzzer extends FuzzBuzz { return resolve() }) }) + */ } async _validate () { for (const [fileName, content] of this.files) { @@ -165,13 +170,34 @@ class HyperdriveFuzzer extends FuzzBuzz { return new Promise((resolve, reject) => { this.drive.stat(fileName, (err, st) => { if (err) return reject(err) - if (!st) return reject(new Error(`File ${fileName} should exist does not exist.`)) + if (!st) return reject(new Error(`File ${fileName} should exist but does not exist.`)) if (st.size !== content.length) return reject(new Error(`Incorrect content length for file ${fileName}.`)) return resolve({ type: 'stat', fileName, stat: st }) }) }) } + statDirectory () { + let selected = this._selectDirectory() + if (!selected) return + + let [dirName, { offset, byteOffset }] = selected + + this.debug(`Statting directory ${dirName}.`) + let fileStat = JSON.stringify(this.files.get(dirName)) + this.debug(` File stat for name: ${fileStat} and typeof ${typeof fileStat}`) + return new Promise((resolve, reject) => { + this.drive.stat(dirName, (err, st) => { + if (err) return reject(err) + this.debug(`Stat for directory ${dirName}: ${JSON.stringify(st)}`) + if (!st) return reject(new Error(`Directory ${dirName} should exist but does not exist.`)) + if (!st.isDirectory()) return reject(new Error(`Stat for directory ${dirName} does not have directory mode`)) + if (st.offset !== offset || st.byteOffset !== byteOffset) return reject(new Error(`Invalid offsets for ${dirName}`)) + return resolve({ type: 'stat', dirName }) + }) + }) + } + existingFileOverwrite () { let selected = this._selectFile() if (!selected) return @@ -190,17 +216,16 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } - writeAndMkdir () { - } - randomReadStream () { let selected = this._selectFile() if (!selected) return let [fileName, content] = selected + return new Promise((resolve, reject) => { let drive = this._validationDrive() let start = this.randomInt(content.length) + this.debug(`Creating random read stream for ${fileName} at start ${start}`) let stream = drive.createReadStream(fileName, { start }) @@ -208,10 +233,44 @@ class HyperdriveFuzzer extends FuzzBuzz { if (err) return reject(err) let buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs) if (!buf.equals(content.slice(start))) return reject(new Error('Read stream does not match content slice.')) + this.debug(`Random read stream for ${fileName} succeeded.`) return resolve() }) }) } + + writeAndMkdir () { + const self = this + + let { name: fileName, content } = this._createFile() + let dirName = this._fileName() + + return new Promise((resolve, reject) => { + let pending = 2 + + let offset = this.drive._contentFeedLength + let byteOffset = this.drive._contentFeedByteLength + + let writeStream = this.drive.createWriteStream(fileName) + writeStream.on('finish', done) + + this.drive.mkdir(dirName, done) + writeStream.end(content) + + function done (err) { + if (err) return reject(err) + if (!--pending) { + self.files.set(fileName, content) + self.debug(`Created directory ${dirName}`) + self.directories.set(dirName, { + offset, + byteOffset + }) + return resolve() + } + } + }) + } } class SparseHyperdriveFuzzer extends HyperdriveFuzzer { @@ -235,26 +294,30 @@ class SparseHyperdriveFuzzer extends HyperdriveFuzzer { module.exports = HyperdriveFuzzer -tape('100000 mixed operations, single drive', async t => { +tape('10000 mixed operations, single drive', async t => { t.plan(1) const fuzz = new HyperdriveFuzzer({ - seed: 'hyperdrive' + seed: 'hyperdrive', + debugging: false }) try { - await fuzz.run(100000) + // let failure = await fuzz.bisect(10000) + // console.log('failure:', failure) + await fuzz.run(10000) t.pass('fuzzing succeeded') } catch (err) { t.error(err, 'no error') } }) -tape.only('10000 mixed operations, replicating drives', async t => { +tape('10000 mixed operations, replicating drives', async t => { t.plan(1) const fuzz = new SparseHyperdriveFuzzer({ - seed: 'hyperdrive2' + seed: 'hyperdrive2', + debugging: false }) try { From bab5eaee411a040390d9709ad079ceadfb750ef2 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 5 Feb 2019 05:05:40 -0800 Subject: [PATCH 014/108] Extract content/storage helpers into lib + more fuzzing --- index.js | 61 ++----------------------------------------------- lib/content.js | 33 ++++++++++++++++++++++++++ lib/storage.js | 38 ++++++++++++++++++++++++++++++ test/fuzzing.js | 13 +++++------ 4 files changed, 79 insertions(+), 66 deletions(-) create mode 100644 lib/content.js create mode 100644 lib/storage.js diff --git a/index.js b/index.js index 5020f41f..076e17fe 100644 --- a/index.js +++ b/index.js @@ -4,10 +4,8 @@ const { EventEmitter } = require('events') const collect = require('stream-collector') const thunky = require('thunky') const unixify = require('unixify') -const raf = require('random-access-file') const mutexify = require('mutexify') const duplexify = require('duplexify') -const sodium = require('sodium-universal') const through = require('through2') const pump = require('pump') @@ -18,6 +16,8 @@ const coreByteStream = require('hypercore-byte-stream') const Stat = require('./lib/stat') const errors = require('./lib/errors') const messages = require('./lib/messages') +const { defaultStorage } = require('./lib/storage') +const { contentKeyPair, contentOptions } = require('./lib/content') class Hyperdrive extends EventEmitter { constructor (storage, key, opts) { @@ -532,63 +532,6 @@ function isObject (val) { return !!val && typeof val !== 'string' && !Buffer.isBuffer(val) } -function wrap (self, storage) { - return { - metadata: function (name, opts) { - return storage.metadata(name, opts, self) - }, - content: function (name, opts) { - return storage.content(name, opts, self) - } - } -} - -function defaultStorage (self, storage, opts) { - var folder = '' - - if (typeof storage === 'object' && storage) return wrap(self, storage) - - if (typeof storage === 'string') { - folder = storage - storage = raf - } - - return { - metadata: function (name) { - return storage(path.join(folder, 'metadata', name)) - }, - content: function (name) { - return storage(path.join(folder, 'content', name)) - } - } -} - -function contentOptions (self, secretKey) { - return { - sparse: self.sparse || self.latest, - maxRequests: self.maxRequests, - secretKey: secretKey, - storeSecretKey: false, - indexing: self.metadataFeed.writable && self.indexing, - storageCacheSize: self.contentStorageCacheSize - } -} - -function contentKeyPair (secretKey) { - var seed = Buffer.allocUnsafe(sodium.crypto_sign_SEEDBYTES) - var context = Buffer.from('hyperdri', 'utf8') // 8 byte context - var keyPair = { - publicKey: Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES), - secretKey: Buffer.allocUnsafe(sodium.crypto_sign_SECRETKEYBYTES) - } - - sodium.crypto_kdf_derive_from_key(seed, 1, context, secretKey) - sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed) - if (seed.fill) seed.fill(0) - - return keyPair -} - function split (buf) { var list = [] for (var i = 0; i < buf.length; i += 65536) { diff --git a/lib/content.js b/lib/content.js new file mode 100644 index 00000000..8482f635 --- /dev/null +++ b/lib/content.js @@ -0,0 +1,33 @@ +const sodium = require('sodium-universal') + +function contentKeyPair (secretKey) { + var seed = Buffer.allocUnsafe(sodium.crypto_sign_SEEDBYTES) + var context = Buffer.from('hyperdri', 'utf8') // 8 byte context + var keyPair = { + publicKey: Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES), + secretKey: Buffer.allocUnsafe(sodium.crypto_sign_SECRETKEYBYTES) + } + + sodium.crypto_kdf_derive_from_key(seed, 1, context, secretKey) + sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed) + if (seed.fill) seed.fill(0) + + return keyPair +} + +function contentOptions (self, secretKey) { + return { + sparse: self.sparse || self.latest, + maxRequests: self.maxRequests, + secretKey: secretKey, + storeSecretKey: false, + indexing: self.metadataFeed.writable && self.indexing, + storageCacheSize: self.contentStorageCacheSize + } +} + +module.exports = { + contentKeyPair, + contentOptions +} + diff --git a/lib/storage.js b/lib/storage.js new file mode 100644 index 00000000..bb7c630a --- /dev/null +++ b/lib/storage.js @@ -0,0 +1,38 @@ +const path = require('path') + +const raf = require('random-access-file') + +function wrap (self, storage) { + return { + metadata: function (name, opts) { + return storage.metadata(name, opts, self) + }, + content: function (name, opts) { + return storage.content(name, opts, self) + } + } +} + +function defaultStorage (self, storage, opts) { + var folder = '' + + if (typeof storage === 'object' && storage) return wrap(self, storage) + + if (typeof storage === 'string') { + folder = storage + storage = raf + } + + return { + metadata: function (name) { + return storage(path.join(folder, 'metadata', name)) + }, + content: function (name) { + return storage(path.join(folder, 'content', name)) + } + } +} + +module.exports = { + defaultStorage +} diff --git a/test/fuzzing.js b/test/fuzzing.js index 45de6b29..0a62bf22 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -48,8 +48,8 @@ class HyperdriveFuzzer extends FuzzBuzz { return char } _fileName () { - let depth = Math.max(this.randomInt(MAX_PATH_DEPTH), 1) do { + let depth = Math.max(this.randomInt(MAX_PATH_DEPTH), 1) var name = (new Array(depth)).fill(0).map(() => this._validChar()).join('/') } while (this.files.get(name) || this.directories.get(name)) return name @@ -127,6 +127,7 @@ class HyperdriveFuzzer extends FuzzBuzz { async call (ops) { + if (!this._counter) this._counter = 0 let res = await super.call(ops) this.log.push(res) } @@ -294,7 +295,7 @@ class SparseHyperdriveFuzzer extends HyperdriveFuzzer { module.exports = HyperdriveFuzzer -tape('10000 mixed operations, single drive', async t => { +tape('20000 mixed operations, single drive', async t => { t.plan(1) const fuzz = new HyperdriveFuzzer({ @@ -303,16 +304,14 @@ tape('10000 mixed operations, single drive', async t => { }) try { - // let failure = await fuzz.bisect(10000) - // console.log('failure:', failure) - await fuzz.run(10000) + await fuzz.run(20000) t.pass('fuzzing succeeded') } catch (err) { t.error(err, 'no error') } }) -tape('10000 mixed operations, replicating drives', async t => { +tape('20000 mixed operations, replicating drives', async t => { t.plan(1) const fuzz = new SparseHyperdriveFuzzer({ @@ -321,7 +320,7 @@ tape('10000 mixed operations, replicating drives', async t => { }) try { - await fuzz.run(10000) + await fuzz.run(20000) t.pass('fuzzing succeeded') } catch (err) { t.error(err, 'no error') From 401be4857c57e072700215cd301ab03c552c8ef5 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 5 Feb 2019 05:12:45 -0800 Subject: [PATCH 015/108] Changed error message for ENOENT --- index.js | 1 - lib/errors.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/index.js b/index.js index 076e17fe..8bee4c9b 100644 --- a/index.js +++ b/index.js @@ -447,7 +447,6 @@ class Hyperdrive extends EventEmitter { this._db.del(name, (err, node) => { if (err) return cb(err) if (!node) return cb(new errors.FileNotFound(name)) - // TODO: Need to check if it's a directory, and the directory was not found return cb(null) }) }) diff --git a/lib/errors.js b/lib/errors.js index 40a57236..e04cc8ef 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -2,7 +2,7 @@ const CustomError = require('custom-error-class') class FileNotFound extends CustomError { constructor (fileName) { - super(`File '${fileName}' not found.`) + super(`No such file or directory: '${fileName}'.`) this.code = 'ENOENT' this.errno = 2 } From 0fce43105b287a4336d133f4a1ae88a028aa810f Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 5 Feb 2019 06:09:43 -0800 Subject: [PATCH 016/108] Reuse ensureContent in initialize + more fuzzing --- index.js | 41 +++++++++++++++++++++++------------------ test/fuzzing.js | 32 ++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/index.js b/index.js index 8bee4c9b..023d0dd9 100644 --- a/index.js +++ b/index.js @@ -114,13 +114,8 @@ class Hyperdrive extends EventEmitter { * The first time the hyperdrive is created, we initialize both the db (metadata feed) and the content feed here. */ function initialize (keyPair) { - self.contentFeed = hypercore(self._storages.content, keyPair.publicKey, self._contentOpts) - self.contentFeed.on('error', function (err) { - self.emit('error', err) - }) - self.contentFeed.ready(function (err) { + self._ensureContent(keyPair.publicKey, err => { if (err) return cb(err) - self._db = hypertrie(null, { feed: self.metadataFeed, metadata: self.contentFeed.key, @@ -147,7 +142,7 @@ class Hyperdrive extends EventEmitter { if (self.metadataFeed.writable) { self._db.ready(err => { if (err) return done(err) - self._ensureContent(done) + self._ensureContent(null, done) }) } else { self._db.ready(done) @@ -170,28 +165,38 @@ class Hyperdrive extends EventEmitter { } } - _ensureContent (cb) { - this._db.getMetadata((err, contentKey) => { - if (err) return cb(err) + _ensureContent (publicKey, cb) { + let self = this + + if (publicKey) return onkey(publicKey) + else loadkey() + + function loadkey () { + self._db.getMetadata((err, contentKey) => { + if (err) return cb(err) + return onkey(contentKey) + }) + } - this.contentFeed = hypercore(this._storages.content, contentKey, this._contentOpts) - this.contentFeed.ready(err => { + function onkey (publicKey) { + self.contentFeed = hypercore(self._storages.content, publicKey, self._contentOpts) + self.contentFeed.ready(err => { if (err) return cb(err) - this._contentFeedByteLength = this.contentFeed.byteLength - this._contentFeedLength = this.contentFeed.length + self._contentFeedByteLength = self.contentFeed.byteLength + self._contentFeedLength = self.contentFeed.length - this.contentFeed.on('error', err => this.emit('error', err)) + self.contentFeed.on('error', err => self.emit('error', err)) return cb(null) }) - }) + } } _contentReady (cb) { this.ready(err => { if (err) return cb(err) if (this.contentFeed) return cb(null) - this._ensureContent(cb) + this._ensureContent(null, cb) }) } @@ -360,7 +365,7 @@ class Hyperdrive extends EventEmitter { name = unixify(name) - this.ready(err => { + this.contentReady(err => { if (err) return cb(err) let st = Stat.directory({ ...opts, diff --git a/test/fuzzing.js b/test/fuzzing.js index 0a62bf22..ec7a65a7 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -79,7 +79,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.log = [] return new Promise((resolve, reject) => { - this.drive.ready(err => { + this.drive.contentReady(err => { if (err) return reject(err) return resolve() }) @@ -278,13 +278,20 @@ class SparseHyperdriveFuzzer extends HyperdriveFuzzer { constructor(opts) { super(opts) } - _setup () { - return super._setup().then(() => { - this.remoteDrive = create(this.drive.key) + async _setup () { + await super._setup() + + this.remoteDrive = create(this.drive.key) + + return new Promise((resolve, reject) => { this.remoteDrive.ready(err => { if (err) throw err let s1 = this.remoteDrive.replicate({ live: true }) s1.pipe(this.drive.replicate({ live: true })).pipe(s1) + this.remoteDrive.contentReady(err => { + if (err) return reject(err) + return resolve() + }) }) }) } @@ -326,3 +333,20 @@ tape('20000 mixed operations, replicating drives', async t => { t.error(err, 'no error') } }) + +tape('100 quick validations (initialization timing)', async t => { + t.plan(1) + + try { + for (let i = 0; i < 100; i++) { + const fuzz = new SparseHyperdriveFuzzer({ + seed: 'iteration #' + i, + debugging: false + }) + await fuzz.run(100) + } + t.pass('fuzzing suceeded') + } catch (err) { + t.error(err, 'no error') + } +}) From 6b87dcacac8dd310d05cb54b1fc0f3d601ef82c6 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 6 Feb 2019 03:57:29 -0800 Subject: [PATCH 017/108] Added missing lib/keys.js + updated docs + end handling in createReadStream --- README.md | 291 ++++++++++++++++++++++++++++++++++++++++- index.js | 13 +- lib/keys.js | 22 ++++ test/helpers/create.js | 4 +- test/storage.js | 18 +-- 5 files changed, 330 insertions(+), 18 deletions(-) create mode 100644 lib/keys.js diff --git a/README.md b/README.md index ab33cd61..feeaa6b8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,289 @@ -# hypertrie-drive -A version of hyperdrive backed by hypertrie +# Hyperdrive + +#### *Note*: This is a prerelease version of Hyperdrive that's backed by [Hypertrie](https://github.com/mafintosh/hypertrie) +#### This version is not yet API-complete. + +Hyperdrive is a secure, real time distributed file system + +``` js +npm install hyperdrive +``` + +[![Build Status](https://travis-ci.org/mafintosh/hyperdrive.svg?branch=master)](https://travis-ci.org/mafintosh/hyperdrive) + +## Usage + +Hyperdrive aims to implement the same API as Node.js' core fs module. + +``` js +var hyperdrive = require('hyperdrive') +var archive = hyperdrive('./my-first-hyperdrive') // content will be stored in this folder + +archive.writeFile('/hello.txt', 'world', function (err) { + if (err) throw err + archive.readdir('/', function (err, list) { + if (err) throw err + console.log(list) // prints ['hello.txt'] + archive.readFile('/hello.txt', 'utf-8', function (err, data) { + if (err) throw err + console.log(data) // prints 'world' + }) + }) +}) +``` + +A big difference is that you can replicate the file system to other computers! All you need is a stream. + +``` js +var net = require('net') + +// ... on one machine + +var server = net.createServer(function (socket) { + socket.pipe(archive.replicate()).pipe(socket) +}) + +server.listen(10000) + +// ... on another + +var clonedArchive = hyperdrive('./my-cloned-hyperdrive', origKey) +var socket = net.connect(10000) + +socket.pipe(clonedArchive.replicate()).pipe(socket) +``` + +It also comes with build in versioning and real time replication. See more below. + +## API + +#### `var archive = hyperdrive(storage, [key], [options])` + +Create a new hyperdrive. + +The `storage` parameter defines how the contents of the archive will be stored. It can be one of the following, depending on how much control you require over how the archive is stored. + +- If you pass in a string, the archive content will be stored in a folder at the given path. +- You can also pass in a function. This function will be called with the name of each of the required files for the archive, and needs to return a [`random-access-storage`](https://github.com/random-access-storage/) instance. +- If you require complete control, you can also pass in an object containing a `metadata` and a `content` field. Both of these need to be functions, and are called with the following arguments: + + - `name`: the name of the file to be stored + - `opts` + - `key`: the [feed key](https://github.com/mafintosh/hypercore#feedkey) of the underlying Hypercore instance + - `discoveryKey`: the [discovery key](https://github.com/mafintosh/hypercore#feeddiscoverykey) of the underlying Hypercore instance + - `archive`: the current Hyperdrive instance + + The functions need to return a a [`random-access-storage`](https://github.com/random-access-storage/) instance. + +Options include: + +``` js +{ + sparse: true, // only download data on content feed when it is specifically requested + sparseMetadata: true // only download data on metadata feed when requested + metadataStorageCacheSize: 65536 // how many entries to use in the metadata hypercore's LRU cache + contentStorageCacheSize: 65536 // how many entries to use in the content hypercore's LRU cache +} +``` + +Note that a cloned hyperdrive archive can be "sparse". Usually (by setting `sparse: true`) this means that the content is not downloaded until you ask for it, but the entire metadata feed is still downloaded. If you want a _very_ sparse archive, where even the metadata feed is not downloaded until you request it, then you should _also_ set `sparseMetadata: true`. + +#### `var stream = archive.replicate([options])` + +Replicate this archive. Options include + +``` js +{ + live: false, // keep replicating + download: true, // download data from peers? + upload: true // upload data to peers? +} +``` + +#### `archive.version` + +Get the current version of the archive (incrementing number). + +#### `archive.key` + +The public key identifying the archive. + +#### `archive.discoveryKey` + +A key derived from the public key that can be used to discovery other peers sharing this archive. + +#### `archive.writable` + +A boolean indicating whether the archive is writable. + +#### `archive.on('ready')` + +Emitted when the archive is fully ready and all properties has been populated. + +#### `archive.on('error', err)` + +Emitted when a critical error during load happened. + +#### `var oldDrive = archive.checkout(version, [opts])` + +Checkout a readonly copy of the archive at an old version. Options are used to configure the `oldDrive`: + +```js +{ + metadataStorageCacheSize: 65536 // how many entries to use in the metadata hypercore's LRU cache + contentStorageCacheSize: 65536 // how many entries to use in the content hypercore's LRU cache + treeCacheSize: 65536 // how many entries to use in the append-tree's LRU cache +} +``` + +#### `archive.download([path], [callback])` + +Download all files in path of current version. +If no path is specified this will download all files. + +You can use this with `.checkout(version)` to download a specific version of the archive. + +``` js +archive.checkout(version).download() +``` + +#### `var stream = archive.createReadStream(name, [options])` + +Read a file out as a stream. Similar to fs.createReadStream. + +Options include: + +``` js +{ + start: optionalByteOffset, // similar to fs + end: optionalInclusiveByteEndOffset, // similar to fs + length: optionalByteLength +} +``` + +#### `archive.readFile(name, [options], callback)` + +Read an entire file into memory. Similar to fs.readFile. + +Options can either be an object or a string + +Options include: +```js +{ + encoding: string + cached: true|false // default: false +} +``` +or a string can be passed as options to simply set the encoding - similar to fs. + +If `cached` is set to `true`, this function returns results only if they have already been downloaded. + +#### `var stream = archive.createWriteStream(name, [options])` + +Write a file as a stream. Similar to fs.createWriteStream. +If `options.cached` is set to `true`, this function returns results only if they have already been downloaded. + +#### `archive.writeFile(name, buffer, [options], [callback])` + +Write a file from a single buffer. Similar to fs.writeFile. + +#### `archive.unlink(name, [callback])` + +Unlinks (deletes) a file. Similar to fs.unlink. + +#### `archive.mkdir(name, [options], [callback])` + +Explictly create an directory. Similar to fs.mkdir + +#### `archive.rmdir(name, [callback])` + +Delete an empty directory. Similar to fs.rmdir. + +#### `archive.readdir(name, [options], [callback])` + +Lists a directory. Similar to fs.readdir. + +Options include: + +``` js +{ + cached: true|false, // default: false +} +``` + +If `cached` is set to `true`, this function returns results from the local version of the archive’s append-tree. Default behavior is to fetch the latest remote version of the archive before returning list of directories. + +#### `archive.stat(name, [options], callback)` + +Stat an entry. Similar to fs.stat. Sample output: + +``` +Stat { + dev: 0, + nlink: 1, + rdev: 0, + blksize: 0, + ino: 0, + mode: 16877, + uid: 0, + gid: 0, + size: 0, + offset: 0, + blocks: 0, + atime: 2017-04-10T18:59:00.147Z, + mtime: 2017-04-10T18:59:00.147Z, + ctime: 2017-04-10T18:59:00.147Z, + linkname: undefined } +``` + +The output object includes methods similar to fs.stat: + +``` js +var stat = archive.stat('/hello.txt') +stat.isDirectory() +stat.isFile() +``` + +Options include: +```js +{ + wait: true|false // default: true +} +``` + +If `wait` is set to `true`, this function will wait for data to be downloaded. If false, will return an error. + +#### `archive.lstat(name, [options], callback)` + +Stat an entry but do not follow symlinks. Similar to fs.lstat. + +Options include: +```js +{ + wait: true|false // default: true +} +``` + +If `wait` is set to `true`, this function will wait for data to be downloaded. If false, will return an error. + +#### `archive.access(name, [options], callback)` + +Similar to fs.access. + +Options include: +```js +{ + wait: true|false // default: true +} +``` + +If `wait` is set to `true`, this function will wait for data to be downloaded. If false, will return an error. + +#### `archive.close(fd, [callback])` + +Close a file. Similar to fs.close. + +#### `archive.close([callback])` + +Closes all open resources used by the archive. +The archive should no longer be used after calling this. diff --git a/index.js b/index.js index 023d0dd9..fbe63051 100644 --- a/index.js +++ b/index.js @@ -19,6 +19,8 @@ const messages = require('./lib/messages') const { defaultStorage } = require('./lib/storage') const { contentKeyPair, contentOptions } = require('./lib/content') +module.exports = (...args) => new Hyperdrive(...args) + class Hyperdrive extends EventEmitter { constructor (storage, key, opts) { super() @@ -33,12 +35,13 @@ class Hyperdrive extends EventEmitter { this.discoveryKey = null this.live = true this.latest = !!opts.latest + this.sparse = !!opts.sparse this._storages = defaultStorage(this, storage, opts) this.metadataFeed = opts.metadataFeed || hypercore(this._storages.metadata, key, { secretKey: opts.secretKey, - sparse: opts.sparseMetadata, + sparse: !!opts.sparseMetadata, createIfMissing: opts.createIfMissing, storageCacheSize: opts.metadataStorageCacheSize, valueEncoding: 'binary' @@ -46,6 +49,7 @@ class Hyperdrive extends EventEmitter { this._db = opts.db this.contentFeed = opts.contentFeed || null this.storage = storage + this.contentStorageCacheSize = opts.contentStorageCacheSize this._contentOpts = null this._contentFeedLength = null @@ -205,6 +209,7 @@ class Hyperdrive extends EventEmitter { name = unixify(name) + const length = typeof opts.end === 'number' ? 1 + opts.end - (opts.start || 0) : typeof opts.length === 'number' ? opts.length : -1 const stream = coreByteStream({ ...opts, highWaterMark: opts.highWaterMark || 64 * 1024 @@ -219,8 +224,8 @@ class Hyperdrive extends EventEmitter { st = st.value - let byteOffset = (opts.start) ? st.byteOffset + opts.start : st.byteOffset - let byteLength = (opts.start) ? st.size - opts.start : st.size + let byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset + let byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) stream.start({ feed: this.contentFeed, @@ -530,8 +535,6 @@ class Hyperdrive extends EventEmitter { } } -module.exports = Hyperdrive - function isObject (val) { return !!val && typeof val !== 'string' && !Buffer.isBuffer(val) } diff --git a/lib/keys.js b/lib/keys.js new file mode 100644 index 00000000..fd327c5f --- /dev/null +++ b/lib/keys.js @@ -0,0 +1,22 @@ +const sodium = require('sodium-universal') + +function contentKeyPair (secretKey) { + var seed = Buffer.allocUnsafe(sodium.crypto_sign_SEEDBYTES) + var context = Buffer.from('hyperdri', 'utf8') // 8 byte context + var keyPair = { + publicKey: Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES), + secretKey: Buffer.allocUnsafe(sodium.crypto_sign_SECRETKEYBYTES) + } + + sodium.crypto_kdf_derive_from_key(seed, 1, context, secretKey) + sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed) + if (seed.fill) seed.fill(0) + + return keyPair +} + +module.exports = { + contentKeyPair, + contentOptions +} + diff --git a/test/helpers/create.js b/test/helpers/create.js index 7803e35b..65dbf151 100644 --- a/test/helpers/create.js +++ b/test/helpers/create.js @@ -1,6 +1,6 @@ var ram = require('random-access-memory') -var Hyperdrive = require('../../') +var hyperdrive = require('../../') module.exports = function (key, opts) { - return new Hyperdrive(ram, key, opts) + return hyperdrive(ram, key, opts) } diff --git a/test/storage.js b/test/storage.js index 05d5598e..1b3f81c6 100644 --- a/test/storage.js +++ b/test/storage.js @@ -1,7 +1,7 @@ -var tape = require('tape') -var tmp = require('temporary-directory') -var create = require('./helpers/create') -var Hyperdrive = require('..') +const tape = require('tape') +const tmp = require('temporary-directory') +const create = require('./helpers/create') +const hyperdrive = require('..') tape('ram storage', function (t) { var archive = create() @@ -16,7 +16,7 @@ tape('ram storage', function (t) { tape('dir storage with resume', function (t) { tmp(function (err, dir, cleanup) { t.ifError(err) - var archive = new Hyperdrive(dir) + var archive = hyperdrive(dir) archive.ready(function () { t.ok(archive.metadataFeed.writable, 'archive metadata is writable') t.ok(archive.contentFeed.writable, 'archive content is writable') @@ -24,7 +24,7 @@ tape('dir storage with resume', function (t) { archive.close(function (err) { t.ifError(err) - var archive2 = new Hyperdrive(dir) + var archive2 = hyperdrive(dir) archive2.ready(function (err) { t.ok(archive2.metadataFeed.writable, 'archive2 metadata is writable') t.ok(archive2.contentFeed.writable, 'archive2 content is writable') @@ -46,7 +46,7 @@ tape('dir storage for non-writable archive', function (t) { tmp(function (err, dir, cleanup) { t.ifError(err) - var clone = new Hyperdrive(dir, src.key) + var clone = hyperdrive(dir, src.key) clone.on('content', function () { t.ok(!clone.metadataFeed.writable, 'clone metadata not writable') t.ok(!clone.contentFeed.writable, 'clone content not writable') @@ -65,7 +65,7 @@ tape('dir storage for non-writable archive', function (t) { tape('dir storage without permissions emits error', function (t) { t.plan(1) - var archive = new Hyperdrive('/') + var archive = hyperdrive('/') archive.on('error', function (err) { t.ok(err, 'got error') }) @@ -76,7 +76,7 @@ tape('write and read (sparse)', function (t) { tmp(function (err, dir, cleanup) { t.ifError(err) - var archive = new Hyperdrive(dir) + var archive = hyperdrive(dir) archive.on('ready', function () { var clone = create(archive.key, {sparse: true}) clone.on('ready', function () { From 792185d9a414084bf6a3c318bb8a7c2578f1a19a Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 6 Feb 2019 04:17:10 -0800 Subject: [PATCH 018/108] Add length to randomReadStream fuzz test --- test/fuzzing.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/fuzzing.js b/test/fuzzing.js index ec7a65a7..b8cb277d 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -226,14 +226,16 @@ class HyperdriveFuzzer extends FuzzBuzz { return new Promise((resolve, reject) => { let drive = this._validationDrive() let start = this.randomInt(content.length) - this.debug(`Creating random read stream for ${fileName} at start ${start}`) + let length = this.randomInt(content.length - start) + this.debug(`Creating random read stream for ${fileName} at start ${start} with length ${length}`) let stream = drive.createReadStream(fileName, { - start + start, + length }) collect(stream, (err, bufs) => { if (err) return reject(err) let buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs) - if (!buf.equals(content.slice(start))) return reject(new Error('Read stream does not match content slice.')) + if (!buf.equals(content.slice(start, start + length))) return reject(new Error('Read stream does not match content slice.')) this.debug(`Random read stream for ${fileName} succeeded.`) return resolve() }) From 483e72a4962336c1159e913208b70bdd34718158 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 6 Feb 2019 04:20:32 -0800 Subject: [PATCH 019/108] Update deps --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 13e7c52c..bc4a0df2 100644 --- a/package.json +++ b/package.json @@ -23,9 +23,9 @@ "dependencies": { "custom-error-class": "^1.0.0", "duplexify": "^3.7.1", - "hypercore": "^6.24.0", - "hypercore-byte-stream": "^1.0.1", - "hypertrie": "^3.3.0", + "hypercore": "^6.25.0", + "hypercore-byte-stream": "^1.0.2", + "hypertrie": "^3.3.1", "mutexify": "^1.2.0", "pump": "^3.0.0", "stream-collector": "^1.0.1", @@ -34,7 +34,7 @@ "unixify": "^1.0.0" }, "devDependencies": { - "fuzzbuzz": "^1.2.0", + "fuzzbuzz": "^2.0.0", "random-access-memory": "^3.1.1", "sodium-universal": "^2.0.0", "tape": "^4.9.2", From 5427b7fc2d6799c5c404909fb0dee4a776d7ca62 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 6 Feb 2019 04:31:15 -0800 Subject: [PATCH 020/108] Update node versions in .travis.yml --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ecc49787..67e381c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ language: node_js node_js: + - "node" + - "lts/*" + - "10" - "8" - - "6" - - "4" From 4bb6547fe3614b50cbd57b9fd2997481369c02bd Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 7 Feb 2019 03:16:42 -0800 Subject: [PATCH 021/108] Fix bug in close + moved sodium-universal to dependencies --- index.js | 2 +- package.json | 2 +- test/fuzzing.js | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index fbe63051..b5e9c1ef 100644 --- a/index.js +++ b/index.js @@ -520,7 +520,7 @@ class Hyperdrive extends EventEmitter { else cb = fd if (!cb) cb = noop - this.contentReady(err => { + this.ready(err => { if (err) return cb(err) this.metadataFeed.close(err => { if (!this.contentFeed) return cb(err) diff --git a/package.json b/package.json index bc4a0df2..19e15ecc 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "hypertrie": "^3.3.1", "mutexify": "^1.2.0", "pump": "^3.0.0", + "sodium-universal": "^2.0.0", "stream-collector": "^1.0.1", "through2": "^3.0.0", "thunky": "^1.0.3", @@ -36,7 +37,6 @@ "devDependencies": { "fuzzbuzz": "^2.0.0", "random-access-memory": "^3.1.1", - "sodium-universal": "^2.0.0", "tape": "^4.9.2", "temporary-directory": "^1.0.2" } diff --git a/test/fuzzing.js b/test/fuzzing.js index b8cb277d..da93e548 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -127,7 +127,6 @@ class HyperdriveFuzzer extends FuzzBuzz { async call (ops) { - if (!this._counter) this._counter = 0 let res = await super.call(ops) this.log.push(res) } From 73229268a7f8cb7e63b0d794d703e55ba5a0e729 Mon Sep 17 00:00:00 2001 From: Mathias Buus Date: Thu, 7 Feb 2019 12:16:58 +0100 Subject: [PATCH 022/108] file descriptor reads --- index.js | 37 +++++++++++++++++++++- lib/fd.js | 66 +++++++++++++++++++++++++++++++++++++++ test/fd.js | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 lib/fd.js create mode 100644 test/fd.js diff --git a/index.js b/index.js index b5e9c1ef..472c232d 100644 --- a/index.js +++ b/index.js @@ -13,6 +13,7 @@ const hypercore = require('hypercore') const hypertrie = require('hypertrie') const coreByteStream = require('hypercore-byte-stream') +const FD = require('./lib/fd') const Stat = require('./lib/stat') const errors = require('./lib/errors') const messages = require('./lib/messages') @@ -55,6 +56,7 @@ class Hyperdrive extends EventEmitter { this._contentFeedLength = null this._contentFeedByteLength = null this._lock = mutexify() + this._fds = [] this.ready = thunky(this._ready.bind(this)) this.contentReady = thunky(this._contentReady.bind(this)) @@ -204,6 +206,35 @@ class Hyperdrive extends EventEmitter { }) } + open (name, flags, cb) { + name = unixify(name) + + this.contentReady(err => { + if (err) return cb(err) + + this._db.get(name, (err, st) => { + if (err) return cb(err) + if (!st) return cb(new errors.FileNotFound(name)) + + // 20 is arbitrary, just to make the fds > stdio etc + const fd = 20 + this._fds.push(new FD(this.contentFeed, name, st.value)) - 1 + cb(null, fd) + }) + }) + } + + read (fd, buf, offset, len, pos, cb) { + if (typeof pos === 'function') { + cb = pos + pos = null + } + + const desc = this._fds[fd - 20] + if (!desc) return process.nextTick(cb, new Error('Invalid file descriptor')) + if (pos == null) pos = desc.position + desc.read(buf, offset, len, pos, cb) + } + createReadStream (name, opts) { if (!opts) opts = {} @@ -511,7 +542,11 @@ class Hyperdrive extends EventEmitter { } _closeFile (fd, cb) { - // TODO: implement + const desc = this._fds[fd - 20] + if (!desc) return process.nextTick(cb, new Error('Invalid file descriptor')) + this._fds[fd - 20] = null + while (this._fds.length && !this._fds[this._fds.length - 1]) this._fds.pop() + desc.close() process.nextTick(cb, null) } diff --git a/lib/fd.js b/lib/fd.js new file mode 100644 index 00000000..befd93a2 --- /dev/null +++ b/lib/fd.js @@ -0,0 +1,66 @@ +module.exports = class FileDescriptor { + constructor (contentFeed, path, stat) { + this.stat = stat + this.path = path + this.position = 0 + this.blockPosition = 0 + this.blockOffset = 0 + this.contentFeed = contentFeed + } + + read (buffer, offset, len, pos, cb) { + if (this.position === pos) this._read(buffer, offset, len, cb) + else this._seekAndRead(buffer, offset, len, pos, cb) + } + + write (buffer, offset, len, pos, cb) { + throw new Error('Not implemented yet') + } + + close () { + // TODO: undownload inital range + } + + _seekAndRead (buffer, offset, len, pos, cb) { + const start = this.stat.offset + const end = start + this.stat.blocks + + this.contentFeed.seek(pos, { start, end }, (err, blk, offset) => { + if (err) return cb(err) + this.position = pos + this.blockPosition = blk + this.blockOffset = offset + this._read(buffer, offset, len, cb) + }) + } + + _read (buffer, offset, len, cb) { + const buf = buffer.slice(offset, offset + len) + const blkOffset = this.blockOffset + const blk = this.blockPosition + + if ((this.stat.offset + this.stat.blocks) <= blk || blk < this.stat.offset) { + return process.nextTick(cb, null, 0, buffer) + } + + this.contentFeed.get(blk, (err, data) => { + if (err) return cb(err) + if (blkOffset) data = data.slice(blkOffset) + + data.copy(buf) + const read = Math.min(data.length, buf.length) + + if (blk === this.blockPosition && blkOffset === this.blockOffset) { + this.position += read + if (read === data.length) { + this.blockPosition++ + this.blockOffset = 0 + } else { + this.blockOffset = blkOffset + read + } + } + + cb(null, read, buffer) + }) + } +} diff --git a/test/fd.js b/test/fd.js new file mode 100644 index 00000000..dd54a719 --- /dev/null +++ b/test/fd.js @@ -0,0 +1,90 @@ +const tape = require('tape') +const create = require('./helpers/create') + +tape('basic fd read', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + + drive.writeFile('hi', content, function (err) { + t.error(err, 'no error') + + drive.open('hi', 'r', function (err, fd) { + t.error(err, 'no error') + t.same(typeof fd, 'number') + t.ok(fd > 5) + + const underflow = 37 + const buf = Buffer.alloc(content.length - underflow) + let pos = 0 + + drive.read(fd, buf, 0, buf.length, 0, function (err, bytesRead) { + t.error(err, 'no error') + pos += bytesRead + t.same(bytesRead, buf.length, 'filled the buffer') + t.same(buf, content.slice(0, buf.length)) + + drive.read(fd, buf, 0, buf.length, pos, function (err, bytesRead) { + t.error(err, 'no error') + pos += bytesRead + t.same(bytesRead, underflow, 'read missing bytes') + t.same(buf.slice(0, underflow), content.slice(content.length - underflow)) + t.same(pos, content.length, 'read full file') + + drive.read(fd, buf, 0, buf.length, pos, function (err, bytesRead) { + t.error(err, 'no error') + t.same(bytesRead, 0, 'no more to read') + + drive.close(fd, function (err) { + t.error(err, 'no error') + t.end() + }) + }) + }) + }) + }) + }) +}) + +tape('basic fd read with implicit position', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + + drive.writeFile('hi', content, function (err) { + t.error(err, 'no error') + + drive.open('hi', 'r', function (err, fd) { + t.error(err, 'no error') + t.same(typeof fd, 'number') + t.ok(fd > 5) + + const underflow = 37 + const buf = Buffer.alloc(content.length - underflow) + let pos = 0 + + drive.read(fd, buf, 0, buf.length, function (err, bytesRead) { + t.error(err, 'no error') + pos += bytesRead + t.same(bytesRead, buf.length, 'filled the buffer') + t.same(buf, content.slice(0, buf.length)) + + drive.read(fd, buf, 0, buf.length, function (err, bytesRead) { + t.error(err, 'no error') + pos += bytesRead + t.same(bytesRead, underflow, 'read missing bytes') + t.same(buf.slice(0, underflow), content.slice(content.length - underflow)) + t.same(pos, content.length, 'read full file') + + drive.read(fd, buf, 0, buf.length, function (err, bytesRead) { + t.error(err, 'no error') + t.same(bytesRead, 0, 'no more to read') + + drive.close(fd, function (err) { + t.error(err, 'no error') + t.end() + }) + }) + }) + }) + }) + }) +}) From 586b2be0fad2183098c10dfeaa9d62cb9041aad9 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 7 Feb 2019 06:16:26 -0800 Subject: [PATCH 023/108] Add BadFileDescriptor error --- index.js | 2 +- lib/errors.js | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 472c232d..8390e695 100644 --- a/index.js +++ b/index.js @@ -230,7 +230,7 @@ class Hyperdrive extends EventEmitter { } const desc = this._fds[fd - 20] - if (!desc) return process.nextTick(cb, new Error('Invalid file descriptor')) + if (!desc) return process.nextTick(cb, new errors.BadFileDescriptor(fd)) if (pos == null) pos = desc.position desc.read(buf, offset, len, pos, cb) } diff --git a/lib/errors.js b/lib/errors.js index e04cc8ef..64f7749f 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -24,8 +24,17 @@ class PathAlreadyExists extends CustomError { } } +class BadFileDescriptor extends CustomError { + constructor (fd) { + super(`Bad file number: ${fd}`) + this.code = 'EBADF' + this.errno = 9 + } +} + module.exports = { FileNotFound, DirectoryNotEmpty, - PathAlreadyExists + PathAlreadyExists, + BadFileDescriptor } From c9122f109d986ea3b5387cbfaea924012b671cf4 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 12 Feb 2019 12:01:17 -0800 Subject: [PATCH 024/108] martinheidegger's suggested changes + failing FD fuzz test --- index.js | 20 ++++++----- lib/keys.js | 22 ------------- test/fd.js | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ test/fuzzing.js | 32 +++++++++++++++++- 4 files changed, 130 insertions(+), 32 deletions(-) delete mode 100644 lib/keys.js diff --git a/index.js b/index.js index 8390e695..abfd5368 100644 --- a/index.js +++ b/index.js @@ -20,6 +20,9 @@ const messages = require('./lib/messages') const { defaultStorage } = require('./lib/storage') const { contentKeyPair, contentOptions } = require('./lib/content') +// 20 is arbitrary, just to make the fds > stdio etc +const STDIO_CAP = 20 + module.exports = (...args) => new Hyperdrive(...args) class Hyperdrive extends EventEmitter { @@ -47,7 +50,7 @@ class Hyperdrive extends EventEmitter { storageCacheSize: opts.metadataStorageCacheSize, valueEncoding: 'binary' }) - this._db = opts.db + this._db = opts._db this.contentFeed = opts.contentFeed || null this.storage = storage this.contentStorageCacheSize = opts.contentStorageCacheSize @@ -207,6 +210,7 @@ class Hyperdrive extends EventEmitter { } open (name, flags, cb) { + if (typeof flags === 'function') return this.open(name, null, flags) name = unixify(name) this.contentReady(err => { @@ -216,8 +220,7 @@ class Hyperdrive extends EventEmitter { if (err) return cb(err) if (!st) return cb(new errors.FileNotFound(name)) - // 20 is arbitrary, just to make the fds > stdio etc - const fd = 20 + this._fds.push(new FD(this.contentFeed, name, st.value)) - 1 + const fd = STDIO_CAP + this._fds.push(new FD(this.contentFeed, name, st.value)) - 1 cb(null, fd) }) }) @@ -229,7 +232,7 @@ class Hyperdrive extends EventEmitter { pos = null } - const desc = this._fds[fd - 20] + const desc = this._fds[fd - STDIO_CAP] if (!desc) return process.nextTick(cb, new errors.BadFileDescriptor(fd)) if (pos == null) pos = desc.position desc.read(buf, offset, len, pos, cb) @@ -255,8 +258,8 @@ class Hyperdrive extends EventEmitter { st = st.value - let byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset - let byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) + const byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset + const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) stream.start({ feed: this.contentFeed, @@ -281,7 +284,7 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return - let stream = pump( + const stream = pump( this._db.createReadStream(name, opts), through.obj((chunk, enc, cb) => { return cb(null, { @@ -531,12 +534,11 @@ class Hyperdrive extends EventEmitter { } checkout (version, opts) { - const versionedTrie = this._db.checkout(version) opts = { ...opts, metadataFeed: this.metadataFeed, contentFeed: this.contentFeed, - db: versionedTrie, + _db: this._db.checkout(version), } return new Hyperdrive(this.storage, this.key, opts) } diff --git a/lib/keys.js b/lib/keys.js deleted file mode 100644 index fd327c5f..00000000 --- a/lib/keys.js +++ /dev/null @@ -1,22 +0,0 @@ -const sodium = require('sodium-universal') - -function contentKeyPair (secretKey) { - var seed = Buffer.allocUnsafe(sodium.crypto_sign_SEEDBYTES) - var context = Buffer.from('hyperdri', 'utf8') // 8 byte context - var keyPair = { - publicKey: Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES), - secretKey: Buffer.allocUnsafe(sodium.crypto_sign_SECRETKEYBYTES) - } - - sodium.crypto_kdf_derive_from_key(seed, 1, context, secretKey) - sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed) - if (seed.fill) seed.fill(0) - - return keyPair -} - -module.exports = { - contentKeyPair, - contentOptions -} - diff --git a/test/fd.js b/test/fd.js index dd54a719..6701f7ff 100644 --- a/test/fd.js +++ b/test/fd.js @@ -88,3 +88,91 @@ tape('basic fd read with implicit position', function (t) { }) }) }) + +tape('fd read with zero length', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + + drive.writeFile('hi', content, function (err) { + t.error(err, 'no error') + + drive.open('hi', 'r', function (err, fd) { + t.error(err, 'no error') + + const buf = Buffer.alloc(content.length) + + drive.read(fd, buf, 0, 0, function (err, bytesRead) { + t.error(err, 'no error') + t.same(bytesRead, 0) + t.end() + }) + }) + }) +}) + +tape('fd read with out-of-bounds offset', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + + drive.writeFile('hi', content, function (err) { + t.error(err, 'no error') + + drive.open('hi', 'r', function (err, fd) { + t.error(err, 'no error') + + const buf = Buffer.alloc(content.length) + + drive.read(fd, buf, content.length, 10, function (err, bytesRead) { + t.error(err, 'no error') + t.same(bytesRead, 0) + t.end() + }) + }) + }) +}) + +tape('fd read with out-of-bounds length', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + + drive.writeFile('hi', content, function (err) { + t.error(err, 'no error') + + drive.open('hi', 'r', function (err, fd) { + t.error(err, 'no error') + + const buf = Buffer.alloc(content.length) + + drive.read(fd, buf, 0, content.length + 1, function (err, bytesRead) { + t.error(err, 'no error') + t.same(bytesRead, content.length) + t.end() + }) + }) + }) +}) + +tape('fd read of empty drive', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + + drive.open('hi', 'r', function (err, fd) { + t.true(err) + t.same(err.errno, 2) + t.end() + }) +}) + +tape('fd read of invalid file', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + + drive.writeFile('hi', content, function (err) { + t.error(err, 'no error') + drive.open('hello', 'r', function (err, fd) { + t.true(err) + t.same(err.errno, 2) + t.end() + }) + }) +}) diff --git a/test/fuzzing.js b/test/fuzzing.js index da93e548..21f9c1d1 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -21,6 +21,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(3, this.statDirectory) this.add(2, this.deleteInvalidFile) this.add(2, this.randomReadStream) + this.add(2, this.randomFileDescriptor) this.add(1, this.writeAndMkdir) } @@ -221,7 +222,6 @@ class HyperdriveFuzzer extends FuzzBuzz { if (!selected) return let [fileName, content] = selected - return new Promise((resolve, reject) => { let drive = this._validationDrive() let start = this.randomInt(content.length) @@ -241,6 +241,35 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } + randomFileDescriptor () { + let selected = this._selectFile() + if (!selected) return + let [fileName, content] = selected + + let length = this.randomInt(content.length) + let start = this.randomInt(content.length) + let actualLength = Math.min(length, content.length) + let buf = Buffer.alloc(actualLength) + + return new Promise((resolve, reject) => { + let drive = this._validationDrive() + drive.open(fileName, (err, fd) => { + if (err) return reject(err) + + console.log('length:', length, 'start:', start, 'content.length:', content.length) + drive.read(fd, buf, 0, length, start, err => { + if (err) return reject(err) + console.log('buf:', buf, 'content:', content) + if (!buf.equals(content.slice(start, actualLength))) return reject(new Error('File descriptor read does not match slice.')) + this.debug(`Random file descriptor read for ${fileName} succeeded`) + + fd.close() + return resolve() + }) + }) + }) + } + writeAndMkdir () { const self = this @@ -273,6 +302,7 @@ class HyperdriveFuzzer extends FuzzBuzz { } }) } + } class SparseHyperdriveFuzzer extends HyperdriveFuzzer { From cb5e19e8fc62042d95e25bd65a8d2bb2033818f8 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 12 Feb 2019 17:28:33 -0800 Subject: [PATCH 025/108] FD bugs + stateful FD fuzz tests --- index.js | 4 +- lib/fd.js | 12 ++--- package.json | 2 +- test/fuzzing.js | 113 +++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 110 insertions(+), 21 deletions(-) diff --git a/index.js b/index.js index abfd5368..30e9bd56 100644 --- a/index.js +++ b/index.js @@ -544,9 +544,9 @@ class Hyperdrive extends EventEmitter { } _closeFile (fd, cb) { - const desc = this._fds[fd - 20] + const desc = this._fds[fd - STDIO_CAP] if (!desc) return process.nextTick(cb, new Error('Invalid file descriptor')) - this._fds[fd - 20] = null + this._fds[fd - STDIO_CAP] = null while (this._fds.length && !this._fds[this._fds.length - 1]) this._fds.pop() desc.close() process.nextTick(cb, null) diff --git a/lib/fd.js b/lib/fd.js index befd93a2..2f205e10 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -2,8 +2,8 @@ module.exports = class FileDescriptor { constructor (contentFeed, path, stat) { this.stat = stat this.path = path - this.position = 0 - this.blockPosition = 0 + this.position = null + this.blockPosition = stat.offset this.blockOffset = 0 this.contentFeed = contentFeed } @@ -14,22 +14,22 @@ module.exports = class FileDescriptor { } write (buffer, offset, len, pos, cb) { - throw new Error('Not implemented yet') + // TODO: implement } close () { - // TODO: undownload inital range + // TODO: undownload initial range } _seekAndRead (buffer, offset, len, pos, cb) { const start = this.stat.offset const end = start + this.stat.blocks - this.contentFeed.seek(pos, { start, end }, (err, blk, offset) => { + this.contentFeed.seek(this.stat.byteOffset + pos, { start, end }, (err, blk, blockOffset) => { if (err) return cb(err) this.position = pos this.blockPosition = blk - this.blockOffset = offset + this.blockOffset = blockOffset this._read(buffer, offset, len, cb) }) } diff --git a/package.json b/package.json index 19e15ecc..dc0b350a 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "devDependencies": { "fuzzbuzz": "^2.0.0", "random-access-memory": "^3.1.1", - "tape": "^4.9.2", + "tape": "^4.10.0", "temporary-directory": "^1.0.2" } } diff --git a/test/fuzzing.js b/test/fuzzing.js index 21f9c1d1..8ec5aa9b 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -17,11 +17,13 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(10, this.writeFile) this.add(5, this.deleteFile) this.add(5, this.existingFileOverwrite) + this.add(5, this.randomStatefulFileDescriptor) this.add(3, this.statFile) this.add(3, this.statDirectory) this.add(2, this.deleteInvalidFile) this.add(2, this.randomReadStream) - this.add(2, this.randomFileDescriptor) + this.add(2, this.randomStatelessFileDescriptor) + this.add(1, this.createFileDescriptor) this.add(1, this.writeAndMkdir) } @@ -41,6 +43,9 @@ class HyperdriveFuzzer extends FuzzBuzz { _selectDirectory () { return this._select(this.directories) } + _selectFileDescriptor () { + return this._select(this.fds) + } _validChar () { do { @@ -77,6 +82,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.files = new Map() this.directories = new Map() this.streams = new Map() + this.fds = new Map() this.log = [] return new Promise((resolve, reject) => { @@ -134,9 +140,10 @@ class HyperdriveFuzzer extends FuzzBuzz { // START Fuzzing operations - async writeFile () { + writeFile () { let { name, content } = this._createFile() return new Promise((resolve, reject) => { + this.debug(`Writing file ${name} with content ${content.length}`) this.drive.writeFile(name, content, err => { if (err) return reject(err) this.files.set(name, content) @@ -145,11 +152,14 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } - async deleteFile () { + deleteFile () { let selected = this._selectFile() if (!selected) return let fileName = selected[0] + + this.debug(`Deleting valid file: ${fileName}`) + return this._deleteFile(fileName) } @@ -157,6 +167,7 @@ class HyperdriveFuzzer extends FuzzBuzz { let name = this._fileName() while (this.files.get(name)) name = this._fileName() try { + this.debug(`Deleting invalid file: ${name}`) await this._deleteFile(name) } catch (err) { if (err && err.code !== 'ENOENT') throw err @@ -169,6 +180,7 @@ class HyperdriveFuzzer extends FuzzBuzz { let [fileName, content] = selected return new Promise((resolve, reject) => { + this.debug(`Statting file: ${fileName}`) this.drive.stat(fileName, (err, st) => { if (err) return reject(err) if (!st) return reject(new Error(`File ${fileName} should exist but does not exist.`)) @@ -207,6 +219,7 @@ class HyperdriveFuzzer extends FuzzBuzz { let { content: newContent } = this._createFile() return new Promise((resolve, reject) => { + this.debug(`Overwriting existing file: ${fileName}`) let writeStream = this.drive.createWriteStream(fileName) writeStream.on('error', err => reject(err)) writeStream.on('finish', () => { @@ -241,7 +254,7 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } - randomFileDescriptor () { + randomStatelessFileDescriptor () { let selected = this._selectFile() if (!selected) return let [fileName, content] = selected @@ -253,23 +266,99 @@ class HyperdriveFuzzer extends FuzzBuzz { return new Promise((resolve, reject) => { let drive = this._validationDrive() + this.debug(`Random stateless file descriptor read for ${fileName}`) drive.open(fileName, (err, fd) => { if (err) return reject(err) - console.log('length:', length, 'start:', start, 'content.length:', content.length) - drive.read(fd, buf, 0, length, start, err => { + drive.read(fd, buf, 0, length, start, (err, bytesRead) => { if (err) return reject(err) - console.log('buf:', buf, 'content:', content) - if (!buf.equals(content.slice(start, actualLength))) return reject(new Error('File descriptor read does not match slice.')) - this.debug(`Random file descriptor read for ${fileName} succeeded`) + buf = buf.slice(0, bytesRead) + let expected = content.slice(start, start + bytesRead) + if (!buf.equals(expected)) return reject(new Error('File descriptor read does not match slice.')) + drive.close(fd, err => { + if (err) return reject(err) - fd.close() - return resolve() + this.debug(`Random file descriptor read for ${fileName} succeeded`) + + return resolve() + }) }) }) }) } + createFileDescriptor () { + let selected = this._selectFile() + if (!selected) return + let [fileName, content] = selected + + let start = this.randomInt(content.length / 5) + let drive = this._validationDrive() + + this.debug(`Creating FD for file ${fileName} and start: ${start}`) + + return new Promise((resolve, reject) => { + drive.open(fileName, (err, fd) => { + if (err) return reject(err) + this.fds.set(fd, { + pos: start, + started: false, + content + }) + return resolve() + }) + }) + } + + randomStatefulFileDescriptor () { + let selected = this._selectFileDescriptor() + if (!selected) return + let [fd, fdInfo] = selected + + let { content, pos, started } = fdInfo + // Try to get multiple reads of of each fd. + let length = this.randomInt(content.length / 5) + let actualLength = Math.min(length, content.length) + let buf = Buffer.alloc(actualLength) + + this.debug(`Reading from random stateful FD ${fd}`) + + let self = this + + return new Promise((resolve, reject) => { + let drive = this._validationDrive() + + let start = null + if (!started) { + fdInfo.started = true + start = fdInfo.pos + } + + drive.read(fd, buf, 0, length, start, (err, bytesRead) => { + if (err) return reject(err) + + if (!bytesRead && length) { + return close() + } + + buf = buf.slice(0, bytesRead) + let expected = content.slice(pos, pos + bytesRead) + if (!buf.equals(expected)) return reject(new Error('File descriptor read does not match slice.')) + + fdInfo.pos += bytesRead + return resolve() + }) + + function close () { + drive.close(fd, err => { + if (err) return reject(err) + self.fds.delete(fd) + return resolve() + }) + } + }) + } + writeAndMkdir () { const self = this @@ -370,7 +459,7 @@ tape('100 quick validations (initialization timing)', async t => { try { for (let i = 0; i < 100; i++) { - const fuzz = new SparseHyperdriveFuzzer({ + const fuzz = new HyperdriveFuzzer({ seed: 'iteration #' + i, debugging: false }) From 397e94f38d4e82d4b3404031b713134eb442ac80 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 16 Feb 2019 20:19:20 -0800 Subject: [PATCH 026/108] Initial writable FD support --- index.js | 60 ++++++++++--------- lib/errors.js | 15 ++++- lib/fd.js | 153 +++++++++++++++++++++++++++++++++++++++++++++--- schema.proto | 1 + test/fd.js | 17 ++++++ test/fuzzing.js | 16 ++--- 6 files changed, 213 insertions(+), 49 deletions(-) diff --git a/index.js b/index.js index 30e9bd56..e515c2cb 100644 --- a/index.js +++ b/index.js @@ -210,19 +210,11 @@ class Hyperdrive extends EventEmitter { } open (name, flags, cb) { - if (typeof flags === 'function') return this.open(name, null, flags) name = unixify(name) - this.contentReady(err => { + FD.create(this, name, flags, (err, fd) => { if (err) return cb(err) - - this._db.get(name, (err, st) => { - if (err) return cb(err) - if (!st) return cb(new errors.FileNotFound(name)) - - const fd = STDIO_CAP + this._fds.push(new FD(this.contentFeed, name, st.value)) - 1 - cb(null, fd) - }) + cb(null, STDIO_CAP + this._fds.push(fd) - 1) }) } @@ -233,11 +225,23 @@ class Hyperdrive extends EventEmitter { } const desc = this._fds[fd - STDIO_CAP] - if (!desc) return process.nextTick(cb, new errors.BadFileDescriptor(fd)) + if (!desc) return process.nextTick(cb, new errors.BadFileDescriptor(`Bad file descriptor: ${fd}`)) if (pos == null) pos = desc.position desc.read(buf, offset, len, pos, cb) } + write (fd, buf, offset, len, pos, cb) { + if (typeof pos === 'function') { + cb = pos + pos = null + } + + const desc = this._fds[fd - STDIO_CAP] + if (!desc) return process.nextTick(cb, new errors.BadFileDescriptor(`Bad file descriptor: ${fd}`)) + if (pos == null) pos = desc.position + desc.write(buf, offset, len, pos, cb) + } + createReadStream (name, opts) { if (!opts) opts = {} @@ -301,12 +305,12 @@ class Hyperdrive extends EventEmitter { createWriteStream (name, opts) { if (!opts) opts = {} - name = unixify(name) const self = this + var release + const proxy = duplexify() - var release = null proxy.setReadable(false) // TODO: support piping through a "split" stream like rabin @@ -315,40 +319,39 @@ class Hyperdrive extends EventEmitter { if (err) return proxy.destroy(err) this._lock(_release => { release = _release - return append() + append() }) }) return proxy function append (err) { - if (err) proxy.destroy(err) + if (err) return proxy.destroy(err) if (proxy.destroyed) return release() - // No one should mutate the content other than us - let byteOffset = self.contentFeed.byteLength - let offset = self.contentFeed.length + const byteOffset = self.contentFeed.byteLength + const offset = self.contentFeed.length self.emit('appending', name, opts) // TODO: revert the content feed if this fails!!!! (add an option to the write stream for this (atomic: true)) const stream = self.contentFeed.createWriteStream() - proxy.on('close', done) - proxy.on('finish', done) + proxy.on('close', ondone) + proxy.on('finish', ondone) proxy.setWritable(stream) proxy.on('prefinish', function () { - var st = Stat.file({ + const stat = Stat.file({ ...opts, - size: self.contentFeed.byteLength - byteOffset, - blocks: self.contentFeed.length - offset, offset: offset, byteOffset: byteOffset, + size: self.contentFeed.byteLength - byteOffset, + blocks: self.contentFeed.length - offset }) proxy.cork() - self._db.put(name, st, function (err) { + self._db.put(name, stat, function (err) { if (err) return proxy.destroy(err) self.emit('append', name, opts) proxy.uncork() @@ -356,9 +359,9 @@ class Hyperdrive extends EventEmitter { }) } - function done () { - proxy.removeListener('close', done) - proxy.removeListener('finish', done) + function ondone () { + proxy.removeListener('close', ondone) + proxy.removeListener('finish', ondone) self._contentFeedLength = self.contentFeed.length self._contentFeedByteLength = self.contentFeed.byteLength release() @@ -548,8 +551,7 @@ class Hyperdrive extends EventEmitter { if (!desc) return process.nextTick(cb, new Error('Invalid file descriptor')) this._fds[fd - STDIO_CAP] = null while (this._fds.length && !this._fds[this._fds.length - 1]) this._fds.pop() - desc.close() - process.nextTick(cb, null) + desc.close(cb) } close (fd, cb) { diff --git a/lib/errors.js b/lib/errors.js index 64f7749f..9a9e2b84 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -25,16 +25,25 @@ class PathAlreadyExists extends CustomError { } class BadFileDescriptor extends CustomError { - constructor (fd) { - super(`Bad file number: ${fd}`) + constructor (msg) { + super(msg) this.code = 'EBADF' this.errno = 9 } } +class InvalidArgument extends CustomError { + constructor (msg) { + super(msg) + this.code = 'EINVAL' + this.errno = 22 + } +} + module.exports = { FileNotFound, DirectoryNotEmpty, PathAlreadyExists, - BadFileDescriptor + BadFileDescriptor, + InvalidArgument } diff --git a/lib/fd.js b/lib/fd.js index 2f205e10..d5e61a44 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -1,31 +1,101 @@ -module.exports = class FileDescriptor { - constructor (contentFeed, path, stat) { - this.stat = stat +const fs = require('fs') +const pump = require('pump') +const errors = require('./errors') + +const { + O_RDONLY, + O_WRONLY, + O_RDWR, + O_CREAT, + O_TRUNC, + O_APPEND, + O_SYNC, + O_EXCL +} = fs.constants +const O_ACCMODE = 3 + +class FileDescriptor { + constructor (drive, path, stat, readable, writable, appending, creating) { + this.drive = drive + this.stat = stat ? stat.value : null this.path = path + + this.readable = readable + this.writable = writable + this.creating = creating + this.appending = appending + this.position = null - this.blockPosition = stat.offset + this.blockPosition = stat ? stat.offset : null this.blockOffset = 0 - this.contentFeed = contentFeed + this._writeStream = null } read (buffer, offset, len, pos, cb) { + if (!this.readable) return cb(new errors.BadFileDescriptor('File descriptor not open for reading.')) if (this.position === pos) this._read(buffer, offset, len, cb) else this._seekAndRead(buffer, offset, len, pos, cb) } write (buffer, offset, len, pos, cb) { - // TODO: implement + if (!this.writable) return cb(new errors.BadFileDescriptor('File descriptor not open for writing.')) + if (!this.stat && !this.creating) { + return cb(new errors.BadFileDescriptor('File descriptor not open in create mode.')) + } + if (this.position !== null && pos !== this.position) { + return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) + } + if (this.appending && pos < this.stat.size) { + return cb(new errors.BadFileDescriptor('Position cannot be less than the file size when appending.')) + } + if (!this._writeStream && pos !== 0) { + return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) + } + + console.log('after checks') + + const self = this + if (this.appending && !this._writeStream) { + // The first write of an appending FD must duplicate the file until random-access writes are supported. + var appendStream = this.drive.createReadStream(this.path) + } + if (!this._writeStream) { + this._writeStream = this.drive.createWriteStream(this.path) + if (appendStream) { + this.position = this.stat.size + return pump(appendStream, this._writeStream, dowrite) + } + } + dowrite() + + function dowrite (err) { + if (err) return cb(err) + const slice = buffer.slice(offset, len) + self._writeStream.write(slice, err => { + if (err) return cb(err) + self.position += slice.length + console.log('slice.length:', slice.length) + return cb(null, slice.length, buffer) + }) + } } - close () { + close (cb) { // TODO: undownload initial range + if (this._writeStream) { + this._writeStream.end(err => { + if (err) return cb(err) + this._writeStream = null + }) + } + process.nextTick(cb, null) } _seekAndRead (buffer, offset, len, pos, cb) { const start = this.stat.offset const end = start + this.stat.blocks - this.contentFeed.seek(this.stat.byteOffset + pos, { start, end }, (err, blk, blockOffset) => { + this.drive.contentFeed.seek(this.stat.byteOffset + pos, { start, end }, (err, blk, blockOffset) => { if (err) return cb(err) this.position = pos this.blockPosition = blk @@ -43,7 +113,7 @@ module.exports = class FileDescriptor { return process.nextTick(cb, null, 0, buffer) } - this.contentFeed.get(blk, (err, data) => { + this.drive.contentFeed.get(blk, (err, data) => { if (err) return cb(err) if (blkOffset) data = data.slice(blkOffset) @@ -64,3 +134,68 @@ module.exports = class FileDescriptor { }) } } + +FileDescriptor.create = function (drive, name, flags, cb) { + try { + flags = toFlagsNumber(flags) + } catch (err) { + return cb(err) + } + + const accmode = flags & O_ACCMODE + const writable = !!(accmode & (O_WRONLY | O_RDWR)) + const readable = accmode === 0 || !!(accmode & O_RDWR) + const appending = !!(flags & O_APPEND) + const creating = !!(flags & O_CREAT) + const canExist = !(flags & O_EXCL) + + drive.contentReady(err => { + if (err) return cb(err) + drive._db.get(name, (err, st) => { + if (err) return cb(err) + if (st && !canExist) return cb(new errors.PathAlreadyExists(name)) + if (!st && (!writable || !creating)) return cb(new errors.FileNotFound(name)) + cb(null, new FileDescriptor(drive, name, st, readable, writable, appending, creating)) + }) + }) +} + +module.exports = FileDescriptor + +// Copied from the Node FS internal utils. +function toFlagsNumber (flags) { + if (typeof flags === 'number') { + return flags + } + + switch (flags) { + case 'r' : return O_RDONLY + case 'rs' : // Fall through. + case 'sr' : return O_RDONLY | O_SYNC + case 'r+' : return O_RDWR + case 'rs+' : // Fall through. + case 'sr+' : return O_RDWR | O_SYNC + + case 'w' : return O_TRUNC | O_CREAT | O_WRONLY + case 'wx' : // Fall through. + case 'xw' : return O_TRUNC | O_CREAT | O_WRONLY | O_EXCL + + case 'w+' : return O_TRUNC | O_CREAT | O_RDWR + case 'wx+': // Fall through. + case 'xw+': return O_TRUNC | O_CREAT | O_RDWR | O_EXCL + + case 'a' : return O_APPEND | O_CREAT | O_WRONLY + case 'ax' : // Fall through. + case 'xa' : return O_APPEND | O_CREAT | O_WRONLY | O_EXCL + case 'as' : // Fall through. + case 'sa' : return O_APPEND | O_CREAT | O_WRONLY | O_SYNC + + case 'a+' : return O_APPEND | O_CREAT | O_RDWR + case 'ax+': // Fall through. + case 'xa+': return O_APPEND | O_CREAT | O_RDWR | O_EXCL + case 'as+': // Fall through. + case 'sa+': return O_APPEND | O_CREAT | O_RDWR | O_SYNC + } + + throw new errors.InvalidArgument(`Invalid value in flags: ${flags}`) +} diff --git a/schema.proto b/schema.proto index 6a89ff2b..46e97adc 100644 --- a/schema.proto +++ b/schema.proto @@ -13,4 +13,5 @@ message Stat { optional uint64 byteOffset = 7; optional uint64 mtime = 8; optional uint64 ctime = 9; + optional bool live = 10; } diff --git a/test/fd.js b/test/fd.js index 6701f7ff..27f30cc1 100644 --- a/test/fd.js +++ b/test/fd.js @@ -176,3 +176,20 @@ tape('fd read of invalid file', function (t) { }) }) }) + +tape.skip('fd basic write, creating file', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + drive.open('hello', 'w+', function (err, fd) { + t.error(err, 'no error') + drive.write(fd, content, 0, content.length, 0, function (err, bytesWritten) { + t.error(err, 'no error') + t.same(bytesWritten, content.length) + drive.readFile('hello', function (err, readContent) { + t.error(err, 'no error') + console.log('readContent:', readContent) + t.true(readContent.equals(content)) + }) + }) + }) +}) diff --git a/test/fuzzing.js b/test/fuzzing.js index 8ec5aa9b..0547d2e9 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -17,13 +17,13 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(10, this.writeFile) this.add(5, this.deleteFile) this.add(5, this.existingFileOverwrite) - this.add(5, this.randomStatefulFileDescriptor) + this.add(5, this.randomStatefulFileDescriptorRead) this.add(3, this.statFile) this.add(3, this.statDirectory) this.add(2, this.deleteInvalidFile) this.add(2, this.randomReadStream) - this.add(2, this.randomStatelessFileDescriptor) - this.add(1, this.createFileDescriptor) + this.add(2, this.randomStatelessFileDescriptorRead) + this.add(1, this.createReadableFileDescriptor) this.add(1, this.writeAndMkdir) } @@ -254,7 +254,7 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } - randomStatelessFileDescriptor () { + randomStatelessFileDescriptorRead () { let selected = this._selectFile() if (!selected) return let [fileName, content] = selected @@ -267,7 +267,7 @@ class HyperdriveFuzzer extends FuzzBuzz { return new Promise((resolve, reject) => { let drive = this._validationDrive() this.debug(`Random stateless file descriptor read for ${fileName}`) - drive.open(fileName, (err, fd) => { + drive.open(fileName, 'r', (err, fd) => { if (err) return reject(err) drive.read(fd, buf, 0, length, start, (err, bytesRead) => { @@ -287,7 +287,7 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } - createFileDescriptor () { + createReadableFileDescriptor () { let selected = this._selectFile() if (!selected) return let [fileName, content] = selected @@ -298,7 +298,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.debug(`Creating FD for file ${fileName} and start: ${start}`) return new Promise((resolve, reject) => { - drive.open(fileName, (err, fd) => { + drive.open(fileName, 'r', (err, fd) => { if (err) return reject(err) this.fds.set(fd, { pos: start, @@ -310,7 +310,7 @@ class HyperdriveFuzzer extends FuzzBuzz { }) } - randomStatefulFileDescriptor () { + randomStatefulFileDescriptorRead () { let selected = this._selectFileDescriptor() if (!selected) return let [fd, fdInfo] = selected From 7db1178a35030424ebd415f6169feb76dc2b5d2c Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 16 Feb 2019 21:21:25 -0800 Subject: [PATCH 027/108] Fix readable fd bug --- index.js | 2 ++ lib/fd.js | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index e515c2cb..59749f12 100644 --- a/index.js +++ b/index.js @@ -230,6 +230,7 @@ class Hyperdrive extends EventEmitter { desc.read(buf, offset, len, pos, cb) } + /* write (fd, buf, offset, len, pos, cb) { if (typeof pos === 'function') { cb = pos @@ -241,6 +242,7 @@ class Hyperdrive extends EventEmitter { if (pos == null) pos = desc.position desc.write(buf, offset, len, pos, cb) } + */ createReadStream (name, opts) { if (!opts) opts = {} diff --git a/lib/fd.js b/lib/fd.js index d5e61a44..89007225 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -33,7 +33,7 @@ class FileDescriptor { read (buffer, offset, len, pos, cb) { if (!this.readable) return cb(new errors.BadFileDescriptor('File descriptor not open for reading.')) - if (this.position === pos) this._read(buffer, offset, len, cb) + if (this.position !== null && this.position === pos) this._read(buffer, offset, len, cb) else this._seekAndRead(buffer, offset, len, pos, cb) } @@ -52,8 +52,6 @@ class FileDescriptor { return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) } - console.log('after checks') - const self = this if (this.appending && !this._writeStream) { // The first write of an appending FD must duplicate the file until random-access writes are supported. @@ -74,7 +72,6 @@ class FileDescriptor { self._writeStream.write(slice, err => { if (err) return cb(err) self.position += slice.length - console.log('slice.length:', slice.length) return cb(null, slice.length, buffer) }) } From 76caf690c1a170c30111471f3eec82737bb6dd64 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 16 Feb 2019 21:56:38 -0800 Subject: [PATCH 028/108] Basic writable fd --- index.js | 2 -- lib/fd.js | 3 ++- test/fd.js | 11 +++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index 59749f12..e515c2cb 100644 --- a/index.js +++ b/index.js @@ -230,7 +230,6 @@ class Hyperdrive extends EventEmitter { desc.read(buf, offset, len, pos, cb) } - /* write (fd, buf, offset, len, pos, cb) { if (typeof pos === 'function') { cb = pos @@ -242,7 +241,6 @@ class Hyperdrive extends EventEmitter { if (pos == null) pos = desc.position desc.write(buf, offset, len, pos, cb) } - */ createReadStream (name, opts) { if (!opts) opts = {} diff --git a/lib/fd.js b/lib/fd.js index 89007225..7ae8412c 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -80,9 +80,10 @@ class FileDescriptor { close (cb) { // TODO: undownload initial range if (this._writeStream) { - this._writeStream.end(err => { + return this._writeStream.end(err => { if (err) return cb(err) this._writeStream = null + return cb(null) }) } process.nextTick(cb, null) diff --git a/test/fd.js b/test/fd.js index 27f30cc1..bd68b5d6 100644 --- a/test/fd.js +++ b/test/fd.js @@ -177,7 +177,7 @@ tape('fd read of invalid file', function (t) { }) }) -tape.skip('fd basic write, creating file', function (t) { +tape.only('fd basic write, creating file', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') drive.open('hello', 'w+', function (err, fd) { @@ -185,10 +185,13 @@ tape.skip('fd basic write, creating file', function (t) { drive.write(fd, content, 0, content.length, 0, function (err, bytesWritten) { t.error(err, 'no error') t.same(bytesWritten, content.length) - drive.readFile('hello', function (err, readContent) { + drive.close(fd, err => { t.error(err, 'no error') - console.log('readContent:', readContent) - t.true(readContent.equals(content)) + drive.readFile('hello', function (err, readContent) { + t.error(err, 'no error') + t.true(readContent.equals(content)) + t.end() + }) }) }) }) From f9e93601a6ea319ddda8b2cb4dc8cfabe7a47ab4 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 16 Feb 2019 22:17:53 -0800 Subject: [PATCH 029/108] writable fd tests --- lib/fd.js | 14 +++++-- test/fd.js | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 5 deletions(-) diff --git a/lib/fd.js b/lib/fd.js index 7ae8412c..a258478d 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -48,7 +48,7 @@ class FileDescriptor { if (this.appending && pos < this.stat.size) { return cb(new errors.BadFileDescriptor('Position cannot be less than the file size when appending.')) } - if (!this._writeStream && pos !== 0) { + if (!this._writeStream && !this.appending && pos !== 0) { return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) } @@ -61,12 +61,18 @@ class FileDescriptor { this._writeStream = this.drive.createWriteStream(this.path) if (appendStream) { this.position = this.stat.size - return pump(appendStream, this._writeStream, dowrite) + // pump does not support the `end` option. + appendStream.pipe(this._writeStream, { end: false }) + + appendStream.on('error', err => this._writeStream.destroy(err)) + this._writeStream.on('error', err => appendStream.destroy(err)) + + return appendStream.on('end', doWrite) } } - dowrite() + doWrite() - function dowrite (err) { + function doWrite (err) { if (err) return cb(err) const slice = buffer.slice(offset, len) self._writeStream.write(slice, err => { diff --git a/test/fd.js b/test/fd.js index bd68b5d6..ff9070a1 100644 --- a/test/fd.js +++ b/test/fd.js @@ -177,7 +177,7 @@ tape('fd read of invalid file', function (t) { }) }) -tape.only('fd basic write, creating file', function (t) { +tape('fd basic write, creating file', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') drive.open('hello', 'w+', function (err, fd) { @@ -196,3 +196,107 @@ tape.only('fd basic write, creating file', function (t) { }) }) }) + +tape('fd basic write, appending file', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + let first = content.slice(0, 2000) + let second = content.slice(2000) + + drive.writeFile('hello', first, err => { + t.error(err, 'no error') + writeSecond() + }) + + function writeSecond () { + drive.open('hello', 'a', function (err, fd) { + t.error(err, 'no error') + drive.write(fd, second, 0, second.length, first.length, function (err, bytesWritten) { + t.error(err, 'no error') + t.same(bytesWritten, second.length) + drive.close(fd, err => { + t.error(err, 'no error') + drive.readFile('hello', function (err, readContent) { + t.error(err, 'no error') + t.true(readContent.equals(content)) + t.end() + }) + }) + }) + }) + } +}) + +tape('fd basic write, overwrite file', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + let first = content.slice(0, 2000) + let second = content.slice(2000) + + drive.writeFile('hello', first, err => { + t.error(err, 'no error') + writeSecond() + }) + + function writeSecond () { + drive.open('hello', 'w', function (err, fd) { + t.error(err, 'no error') + drive.write(fd, second, 0, second.length, 0, function (err, bytesWritten) { + t.error(err, 'no error') + t.same(bytesWritten, second.length) + drive.close(fd, err => { + t.error(err, 'no error') + drive.readFile('hello', function (err, readContent) { + t.error(err, 'no error') + t.true(readContent.equals(second)) + t.end() + }) + }) + }) + }) + } +}) + +tape('fd stateful write', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + let first = content.slice(0, 2000) + let second = content.slice(2000) + + drive.open('hello', 'w', function (err, fd) { + t.error(err, 'no error') + drive.write(fd, first, 0, first.length, 0, function (err) { + t.error(err, 'no error') + drive.write(fd, second, 0, second.length, first.length, function (err) { + t.error(err, 'no error') + drive.close(fd, err => { + t.error(err, 'no error') + drive.readFile('hello', function (err, readContent) { + t.error(err, 'no error') + t.true(readContent.equals(content)) + t.end() + }) + }) + }) + }) + }) +}) + +tape('fd random-access write fails', function (t) { + const drive = create() + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + let first = content.slice(0, 2000) + let second = content.slice(2000) + + drive.open('hello', 'w', function (err, fd) { + t.error(err, 'no error') + drive.write(fd, first, 0, first.length, 0, function (err) { + t.error(err, 'no error') + drive.write(fd, second, 0, second.length, first.length - 500, function (err) { + t.true(err) + t.same(err.errno, 9) + t.end() + }) + }) + }) +}) From 235b79763eb7e8a1020c4c87c88d015a7f8e515a Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 18 Feb 2019 03:37:13 -0800 Subject: [PATCH 030/108] Add factory option --- index.js | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index e515c2cb..8e3ce2a4 100644 --- a/index.js +++ b/index.js @@ -41,9 +41,10 @@ class Hyperdrive extends EventEmitter { this.latest = !!opts.latest this.sparse = !!opts.sparse - this._storages = defaultStorage(this, storage, opts) + this._factory = opts.factory ? storage : null + this._storages = !this.factory ? defaultStorage(this, storage, opts) : null - this.metadataFeed = opts.metadataFeed || hypercore(this._storages.metadata, key, { + this.metadataFeed = opts.metadataFeed || this._createHypercore(this._storages.metadata, key, { secretKey: opts.secretKey, sparse: !!opts.sparseMetadata, createIfMissing: opts.createIfMissing, @@ -80,6 +81,11 @@ class Hyperdrive extends EventEmitter { } } + _createHypercore (storage, key, opts) { + if (this._factory) return this._factory(key, opts) + return hypercore(storage, key, opts) + } + get version () { // TODO: The trie version starts at 1, so the empty hyperdrive version is also 1. This should be 0. return this._db.version @@ -98,8 +104,9 @@ class Hyperdrive extends EventEmitter { this.metadataFeed.ready(err => { if (err) return cb(err) - const keyPair = this.metadataFeed.secretKey ? contentKeyPair(this.metadataFeed.secretKey) : {} - this._contentOpts = contentOptions(this, keyPair.secretKey) + this._contentKeyPair = this.metadataFeed.secretKey ? contentKeyPair(this.metadataFeed.secretKey) : {} + this._contentOpts = contentOptions(this, this._contentKeyPair.secretKey) + this._contentOpts.keyPair = this._contentKeyPair /** * If a db is provided as input, ensure that a contentFeed is also provided, then return (this is a checkout). @@ -113,9 +120,9 @@ class Hyperdrive extends EventEmitter { if (!this.contentFeed || !this.metadataFeed) return cb(new Error('Must provide a db and both content/metadata feeds')) return done(null) } else if (this.metadataFeed.writable && !this.metadataFeed.length) { - initialize(keyPair) + initialize(this._contentKeyPair) } else { - restore(keyPair) + restore(this._contentKeyPair) } }) @@ -188,7 +195,7 @@ class Hyperdrive extends EventEmitter { } function onkey (publicKey) { - self.contentFeed = hypercore(self._storages.content, publicKey, self._contentOpts) + self.contentFeed = self._createHypercore(self._storages.content, publicKey, self._contentOpts) self.contentFeed.ready(err => { if (err) return cb(err) From a9141cfacf56d2376346b6ba605cd220bdf04b22 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 18 Feb 2019 04:09:02 -0800 Subject: [PATCH 031/108] Add updateMetadata --- index.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/index.js b/index.js index 8e3ce2a4..7e95778a 100644 --- a/index.js +++ b/index.js @@ -575,6 +575,23 @@ class Hyperdrive extends EventEmitter { }) } + updateMetadata (name, stat, cb) { + name = unixify(name) + + this.ready(err => { + if (err) return cb(err) + this._db.get(name, (err, st) => { + if (err) return cb(err) + if (!st) return cb(new errors.FileNotFound(name)) + const newStat = Object.assign(st.value, stat) + this._db.put(name, newStat, err => { + if (err) return cb(err) + return cb(null) + }) + }) + }) + } + watch (name, onchange) { name = unixify(name) return this._db.watch(name, onchange) From 5a1845d1218fcd5a975420b3781b045d1175b20b Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 21 Feb 2019 00:27:49 -0800 Subject: [PATCH 032/108] Manual decoding --- index.js | 41 +++++++++++++++++++---------------------- lib/fd.js | 6 +++++- lib/stat.js | 3 +++ package.json | 2 +- schema.proto | 1 - 5 files changed, 28 insertions(+), 25 deletions(-) diff --git a/index.js b/index.js index 7e95778a..b697b297 100644 --- a/index.js +++ b/index.js @@ -135,7 +135,6 @@ class Hyperdrive extends EventEmitter { self._db = hypertrie(null, { feed: self.metadataFeed, metadata: self.contentFeed.key, - valueEncoding: messages.Stat }) self._db.ready(function (err) { @@ -152,8 +151,7 @@ class Hyperdrive extends EventEmitter { */ function restore (keyPair) { self._db = hypertrie(null, { - feed: self.metadataFeed, - valueEncoding: messages.Stat + feed: self.metadataFeed }) if (self.metadataFeed.writable) { self._db.ready(err => { @@ -216,6 +214,21 @@ class Hyperdrive extends EventEmitter { }) } + _update (name, stat, cb) { + name = unixify(name) + + if (err) return cb(err) + this._db.get(name, (err, st) => { + if (err) return cb(err) + if (!st) return cb(new errors.FileNotFound(name)) + const newStat = Object.assign(messages.Stat.decode(st.value), stat) + this._db.put(name, newStat, err => { + if (err) return cb(err) + return cb(null) + }) + }) + } + open (name, flags, cb) { name = unixify(name) @@ -437,7 +450,7 @@ class Hyperdrive extends EventEmitter { ite.next((err, st) => { if (err) return cb(err) if (name !== '/' && !st) return cb(new errors.FileNotFound(name)) - st = Stat.directory() + st = Stat.directory(st) return cb(null, st) }) } @@ -453,7 +466,8 @@ class Hyperdrive extends EventEmitter { this._db.get(name, opts, (err, node) => { if (err) return cb(err) if (!node) return this._statDirectory(name, opts, cb) - cb(null, new Stat(node.value)) + const st = messages.Stat.decode(node.value) + cb(null, new Stat(st)) }) }) } @@ -575,23 +589,6 @@ class Hyperdrive extends EventEmitter { }) } - updateMetadata (name, stat, cb) { - name = unixify(name) - - this.ready(err => { - if (err) return cb(err) - this._db.get(name, (err, st) => { - if (err) return cb(err) - if (!st) return cb(new errors.FileNotFound(name)) - const newStat = Object.assign(st.value, stat) - this._db.put(name, newStat, err => { - if (err) return cb(err) - return cb(null) - }) - }) - }) - } - watch (name, onchange) { name = unixify(name) return this._db.watch(name, onchange) diff --git a/lib/fd.js b/lib/fd.js index a258478d..44e800dd 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -1,6 +1,7 @@ const fs = require('fs') const pump = require('pump') const errors = require('./errors') +const messages = require('./messages') const { O_RDONLY, @@ -17,7 +18,7 @@ const O_ACCMODE = 3 class FileDescriptor { constructor (drive, path, stat, readable, writable, appending, creating) { this.drive = drive - this.stat = stat ? stat.value : null + this.stat = stat this.path = path this.readable = readable @@ -159,6 +160,9 @@ FileDescriptor.create = function (drive, name, flags, cb) { if (err) return cb(err) if (st && !canExist) return cb(new errors.PathAlreadyExists(name)) if (!st && (!writable || !creating)) return cb(new errors.FileNotFound(name)) + + if (st) st = messages.Stat.decode(st.value) + cb(null, new FileDescriptor(drive, name, st, readable, writable, appending, creating)) }) }) diff --git a/lib/stat.js b/lib/stat.js index 2d16de94..fa78b577 100644 --- a/lib/stat.js +++ b/lib/stat.js @@ -1,4 +1,5 @@ // http://man7.org/linux/man-pages/man2/stat.2.html +const messages = require('messages') var DEFAULT_FMODE = (4 | 2 | 0) << 6 | ((4 | 0 | 0) << 3) | (4 | 0 | 0) // rw-r--r-- var DEFAULT_DMODE = (4 | 2 | 1) << 6 | ((4 | 0 | 1) << 3) | (4 | 0 | 1) // rwxr-xr-x @@ -52,11 +53,13 @@ class Stat { } Stat.file = function (data) { + if (data instanceof Buffer) data = messages.Stat.decode(data) data = data || {} data.mode = (data.mode || DEFAULT_FMODE) | Stat.IFREG return new Stat(data) } Stat.directory = function (data) { + if (data instanceof Buffer) data = messages.Stat.decode(data) data = data || {} data.mode = (data.mode || DEFAULT_DMODE) | Stat.IFDIR return new Stat(data) diff --git a/package.json b/package.json index dc0b350a..8c03b487 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "duplexify": "^3.7.1", "hypercore": "^6.25.0", "hypercore-byte-stream": "^1.0.2", - "hypertrie": "^3.3.1", + "hypertrie": "^3.4.0", "mutexify": "^1.2.0", "pump": "^3.0.0", "sodium-universal": "^2.0.0", diff --git a/schema.proto b/schema.proto index 46e97adc..6a89ff2b 100644 --- a/schema.proto +++ b/schema.proto @@ -13,5 +13,4 @@ message Stat { optional uint64 byteOffset = 7; optional uint64 mtime = 8; optional uint64 ctime = 9; - optional bool live = 10; } From 10ebb9fb20e628167b19faf815a48f3b3adcbcc4 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 21 Feb 2019 03:02:47 -0800 Subject: [PATCH 033/108] Better error handling + writable fd fuzz tests --- index.js | 53 ++++++++++++++++++++++-------- lib/fd.js | 70 +++++++++++++++++++++++++++------------- lib/stat.js | 4 --- test/basic.js | 1 + test/fuzzing.js | 85 ++++++++++++++++++++++++++++++++++++++++--------- 5 files changed, 159 insertions(+), 54 deletions(-) diff --git a/index.js b/index.js index b697b297..3477c596 100644 --- a/index.js +++ b/index.js @@ -217,11 +217,15 @@ class Hyperdrive extends EventEmitter { _update (name, stat, cb) { name = unixify(name) - if (err) return cb(err) this._db.get(name, (err, st) => { if (err) return cb(err) if (!st) return cb(new errors.FileNotFound(name)) - const newStat = Object.assign(messages.Stat.decode(st.value), stat) + try { + var decoded = messages.Stat.decode(st.value) + } catch (err) { + return cb(err) + } + const newStat = Object.assign(decoded, stat) this._db.put(name, newStat, err => { if (err) return cb(err) return cb(null) @@ -280,7 +284,11 @@ class Hyperdrive extends EventEmitter { if (err) return stream.destroy(err) if (!st) return stream.destroy(new errors.FileNotFound(name)) - st = st.value + try { + st = messages.Stat.decode(st.value) + } catch (err) { + return stream.destroy(err) + } const byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) @@ -323,7 +331,7 @@ class Hyperdrive extends EventEmitter { return proxy } - createWriteStream (name, opts) { + createWriteStream (name, opts) { if (!opts) opts = {} name = unixify(name) @@ -369,9 +377,14 @@ class Hyperdrive extends EventEmitter { size: self.contentFeed.byteLength - byteOffset, blocks: self.contentFeed.length - offset }) + try { + var encoded = messages.Stat.encode(stat) + } catch (err) { + return proxy.destroy(err) + } proxy.cork() - self._db.put(name, stat, function (err) { + self._db.put(name, encoded, function (err) { if (err) return proxy.destroy(err) self.emit('append', name, opts) proxy.uncork() @@ -429,11 +442,15 @@ class Hyperdrive extends EventEmitter { this.contentReady(err => { if (err) return cb(err) - let st = Stat.directory({ - ...opts, - offset: this._contentFeedLength, - byteOffset: this._contentFeedByteLength - }) + try { + var st = messages.Stat.encode(Stat.directory({ + ...opts, + offset: this._contentFeedLength, + byteOffset: this._contentFeedByteLength + })) + } catch (err) { + return cb(err) + } this._db.put(name, st, { condition: ifNotExists }, cb) @@ -450,8 +467,14 @@ class Hyperdrive extends EventEmitter { ite.next((err, st) => { if (err) return cb(err) if (name !== '/' && !st) return cb(new errors.FileNotFound(name)) - st = Stat.directory(st) - return cb(null, st) + if (st) { + try { + st = messages.Stat.decode(st) + } catch (err) { + return cb(err) + } + } + return cb(null, Stat.directory(st)) }) } @@ -466,7 +489,11 @@ class Hyperdrive extends EventEmitter { this._db.get(name, opts, (err, node) => { if (err) return cb(err) if (!node) return this._statDirectory(name, opts, cb) - const st = messages.Stat.decode(node.value) + try { + var st = messages.Stat.decode(node.value) + } catch (err) { + return cb(err) + } cb(null, new Stat(st)) }) }) diff --git a/lib/fd.js b/lib/fd.js index 44e800dd..2819ad7b 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -29,7 +29,15 @@ class FileDescriptor { this.position = null this.blockPosition = stat ? stat.offset : null this.blockOffset = 0 - this._writeStream = null + + this._err = null + if (this.writable) { + this._writeStream = this.drive.createWriteStream(this.path) + this._writeStream.on('error', err => { + this._err = err + }) + if (this.appending) this._appendStream = this.drive.createReadStream(this.path) + } } read (buffer, offset, len, pos, cb) { @@ -54,44 +62,56 @@ class FileDescriptor { } const self = this - if (this.appending && !this._writeStream) { - // The first write of an appending FD must duplicate the file until random-access writes are supported. - var appendStream = this.drive.createReadStream(this.path) - } - if (!this._writeStream) { - this._writeStream = this.drive.createWriteStream(this.path) - if (appendStream) { - this.position = this.stat.size - // pump does not support the `end` option. - appendStream.pipe(this._writeStream, { end: false }) - appendStream.on('error', err => this._writeStream.destroy(err)) - this._writeStream.on('error', err => appendStream.destroy(err)) + // TODO: This is a temporary (bad) way of supporting appends. + if (this._appendStream) { + this.position = this.stat.size + // pump does not support the `end` option. + this._appendStream.pipe(this._writeStream, { end: false }) - return appendStream.on('end', doWrite) - } + this._appendStream.on('error', err => this._writeStream.destroy(err)) + this._writeStream.on('error', err => this._appendStream.destroy(err)) + + return this._appendStream.on('end', doWrite) } - doWrite() + + return doWrite() function doWrite (err) { + if (self._err) return cb(self._err) + if (self._writeStream.destroyed) return cb(new errors.BadFileDescriptor('Write stream was destroyed.')) + + self._appendStream = null if (err) return cb(err) const slice = buffer.slice(offset, len) - self._writeStream.write(slice, err => { + write(self._writeStream, slice, err => { if (err) return cb(err) self.position += slice.length return cb(null, slice.length, buffer) }) } + + function write(stream, data, cb) { + if (!stream.write(data)) { + stream.once('drain', cb) + } else { + process.nextTick(cb) + } + } } close (cb) { // TODO: undownload initial range if (this._writeStream) { - return this._writeStream.end(err => { - if (err) return cb(err) + if (this._writeStream.destroyed) { this._writeStream = null - return cb(null) - }) + } else { + return this._writeStream.end(err => { + if (err) return cb(err) + this._writeStream = null + return cb(null) + }) + } } process.nextTick(cb, null) } @@ -161,7 +181,13 @@ FileDescriptor.create = function (drive, name, flags, cb) { if (st && !canExist) return cb(new errors.PathAlreadyExists(name)) if (!st && (!writable || !creating)) return cb(new errors.FileNotFound(name)) - if (st) st = messages.Stat.decode(st.value) + if (st) { + try { + st = messages.Stat.decode(st.value) + } catch (err) { + return cb(err) + } + } cb(null, new FileDescriptor(drive, name, st, readable, writable, appending, creating)) }) diff --git a/lib/stat.js b/lib/stat.js index fa78b577..875467b8 100644 --- a/lib/stat.js +++ b/lib/stat.js @@ -1,6 +1,4 @@ // http://man7.org/linux/man-pages/man2/stat.2.html -const messages = require('messages') - var DEFAULT_FMODE = (4 | 2 | 0) << 6 | ((4 | 0 | 0) << 3) | (4 | 0 | 0) // rw-r--r-- var DEFAULT_DMODE = (4 | 2 | 1) << 6 | ((4 | 0 | 1) << 3) | (4 | 0 | 1) // rwxr-xr-x @@ -53,13 +51,11 @@ class Stat { } Stat.file = function (data) { - if (data instanceof Buffer) data = messages.Stat.decode(data) data = data || {} data.mode = (data.mode || DEFAULT_FMODE) | Stat.IFREG return new Stat(data) } Stat.directory = function (data) { - if (data instanceof Buffer) data = messages.Stat.decode(data) data = data || {} data.mode = (data.mode || DEFAULT_DMODE) | Stat.IFDIR return new Stat(data) diff --git a/test/basic.js b/test/basic.js index 30ba388a..86dde142 100644 --- a/test/basic.js +++ b/test/basic.js @@ -7,6 +7,7 @@ tape('write and read', function (t) { archive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') + console.log('reading') archive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') t.same(buf, Buffer.from('world')) diff --git a/test/fuzzing.js b/test/fuzzing.js index 0547d2e9..2a1fc4c5 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -8,7 +8,9 @@ const create = require('./helpers/create') const MAX_PATH_DEPTH = 30 const MAX_FILE_LENGTH = 1e3 const CHARACTERS = 1e3 -const INVALID_CHARS = new Set(['/', '\\', '?', '%', '*', ':', '|', '"', '<', '>', '.', ' ']) +const APPROX_READS_PER_FD = 5 +const APPROX_WRITES_PER_FD = 5 +const INVALID_CHARS = new Set(['/', '\\', '?', '%', '*', ':', '|', '"', '<', '>', '.', ' ', '\n', '\t', '\r']) class HyperdriveFuzzer extends FuzzBuzz { constructor (opts) { @@ -18,6 +20,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(5, this.deleteFile) this.add(5, this.existingFileOverwrite) this.add(5, this.randomStatefulFileDescriptorRead) + this.add(5, this.randomStatefulFileDescriptorWrite) this.add(3, this.statFile) this.add(3, this.statDirectory) this.add(2, this.deleteInvalidFile) @@ -43,8 +46,8 @@ class HyperdriveFuzzer extends FuzzBuzz { _selectDirectory () { return this._select(this.directories) } - _selectFileDescriptor () { - return this._select(this.fds) + _selectReadableFileDescriptor () { + return this._select(this.readable_fds) } _validChar () { @@ -60,9 +63,12 @@ class HyperdriveFuzzer extends FuzzBuzz { } while (this.files.get(name) || this.directories.get(name)) return name } + _content () { + return Buffer.allocUnsafe(this.randomInt(MAX_FILE_LENGTH)).fill(0).map(() => this.randomInt(10)) + } _createFile () { let name = this._fileName() - let content = Buffer.allocUnsafe(this.randomInt(MAX_FILE_LENGTH)).fill(0).map(() => this.randomInt(10)) + let content = this._content() return { name, content } } _deleteFile (name) { @@ -82,7 +88,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.files = new Map() this.directories = new Map() this.streams = new Map() - this.fds = new Map() + this.readable_fds = new Map() this.log = [] return new Promise((resolve, reject) => { @@ -197,15 +203,13 @@ class HyperdriveFuzzer extends FuzzBuzz { let [dirName, { offset, byteOffset }] = selected this.debug(`Statting directory ${dirName}.`) - let fileStat = JSON.stringify(this.files.get(dirName)) - this.debug(` File stat for name: ${fileStat} and typeof ${typeof fileStat}`) return new Promise((resolve, reject) => { this.drive.stat(dirName, (err, st) => { if (err) return reject(err) - this.debug(`Stat for directory ${dirName}: ${JSON.stringify(st)}`) if (!st) return reject(new Error(`Directory ${dirName} should exist but does not exist.`)) if (!st.isDirectory()) return reject(new Error(`Stat for directory ${dirName} does not have directory mode`)) if (st.offset !== offset || st.byteOffset !== byteOffset) return reject(new Error(`Invalid offsets for ${dirName}`)) + this.debug(` Successfully statted directory.`) return resolve({ type: 'stat', dirName }) }) }) @@ -266,7 +270,7 @@ class HyperdriveFuzzer extends FuzzBuzz { return new Promise((resolve, reject) => { let drive = this._validationDrive() - this.debug(`Random stateless file descriptor read for ${fileName}`) + this.debug(`Random stateless file descriptor read for ${fileName}, ${length} starting at ${start}`) drive.open(fileName, 'r', (err, fd) => { if (err) return reject(err) @@ -295,12 +299,11 @@ class HyperdriveFuzzer extends FuzzBuzz { let start = this.randomInt(content.length / 5) let drive = this._validationDrive() - this.debug(`Creating FD for file ${fileName} and start: ${start}`) - return new Promise((resolve, reject) => { + this.debug(`Creating readable FD for file ${fileName} and start: ${start}`) drive.open(fileName, 'r', (err, fd) => { if (err) return reject(err) - this.fds.set(fd, { + this.readable_fds.set(fd, { pos: start, started: false, content @@ -311,13 +314,13 @@ class HyperdriveFuzzer extends FuzzBuzz { } randomStatefulFileDescriptorRead () { - let selected = this._selectFileDescriptor() + let selected = this._selectReadableFileDescriptor() if (!selected) return let [fd, fdInfo] = selected let { content, pos, started } = fdInfo // Try to get multiple reads of of each fd. - let length = this.randomInt(content.length / 5) + let length = this.randomInt(content.length / APPROX_READS_PER_FD) let actualLength = Math.min(length, content.length) let buf = Buffer.alloc(actualLength) @@ -352,7 +355,57 @@ class HyperdriveFuzzer extends FuzzBuzz { function close () { drive.close(fd, err => { if (err) return reject(err) - self.fds.delete(fd) + self.readable_fds.delete(fd) + return resolve() + }) + } + }) + } + + randomStatefulFileDescriptorWrite () { + let append = !!this.randomInt(1) + let flags = append ? 'a' : 'w+' + + if (append) { + let selected = this._selectFile() + if (!selected) return + var [fileName, content] = selected + var pos = content.length + } else { + fileName = this._fileName() + content = Buffer.alloc(0) + pos = 0 + } + + const bufs = new Array(this.randomInt(APPROX_WRITES_PER_FD - 1)).fill(0).map(() => this._content()) + const self = this + + let count = 0 + + return new Promise((resolve, reject) => { + this.debug(`Writing stateful file descriptor for fileName ${fileName} with flags ${flags} and buffers ${bufs.length}`) + this.drive.open(fileName, flags, (err, fd) => { + if (err) return reject(err) + if (!bufs.length) return close(fd) + return writeNext(fd) + }) + + function writeNext(fd) { + let next = bufs[count] + self.debug(` Writing content with length ${next.length} to FD ${fd} at pos: ${pos}`) + self.drive.write(fd, next, 0, next.length, pos, (err, bytesWritten) => { + if (err) return reject(err) + pos += bytesWritten + bufs[count] = next.slice(0, bytesWritten) + if (++count === bufs.length) return close(fd) + return writeNext(fd) + }) + } + + function close (fd) { + self.drive.close(fd, err => { + if (err) return reject(err) + self.files.set(fileName, Buffer.concat([content, ...bufs])) return resolve() }) } @@ -366,6 +419,8 @@ class HyperdriveFuzzer extends FuzzBuzz { let dirName = this._fileName() return new Promise((resolve, reject) => { + this.debug(`Writing ${fileName} and making dir ${dirName} simultaneously`) + let pending = 2 let offset = this.drive._contentFeedLength From 80d8cc83977b7b99e686f00a32c6d5d2ae168b46 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 21 Feb 2019 03:09:13 -0800 Subject: [PATCH 034/108] Fixed bug in _update --- index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 3477c596..2bae32d7 100644 --- a/index.js +++ b/index.js @@ -222,11 +222,12 @@ class Hyperdrive extends EventEmitter { if (!st) return cb(new errors.FileNotFound(name)) try { var decoded = messages.Stat.decode(st.value) + const newStat = Object.assign(decoded, stat) + var encoded = messages.Stat.encode(newStat) } catch (err) { return cb(err) } - const newStat = Object.assign(decoded, stat) - this._db.put(name, newStat, err => { + this._db.put(name, encoded, err => { if (err) return cb(err) return cb(null) }) From 4beac45a35164570cd6a05e5df6f8a73d200f021 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 21 Feb 2019 03:16:13 -0800 Subject: [PATCH 035/108] Decode value in _statDirectory --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 2bae32d7..56d931cd 100644 --- a/index.js +++ b/index.js @@ -470,7 +470,7 @@ class Hyperdrive extends EventEmitter { if (name !== '/' && !st) return cb(new errors.FileNotFound(name)) if (st) { try { - st = messages.Stat.decode(st) + st = messages.Stat.decode(st.value) } catch (err) { return cb(err) } From 223bf00bbddd71b6eed99249782c150e1e695467 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 21 Feb 2019 03:22:41 -0800 Subject: [PATCH 036/108] createDirectoryStream decoding error --- index.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 56d931cd..45185723 100644 --- a/index.js +++ b/index.js @@ -320,9 +320,14 @@ class Hyperdrive extends EventEmitter { const stream = pump( this._db.createReadStream(name, opts), through.obj((chunk, enc, cb) => { + try { + var stat = messages.Stat.decode(chunk.value) + } catch (err) { + return cb(err) + } return cb(null, { path: chunk.key, - stat: new Stat(chunk.value) + stat: new Stat(stat) }) }) ) From 6221c30345ec96f29c240719a34c036ef118007f Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 21 Feb 2019 03:55:27 -0800 Subject: [PATCH 037/108] Fixed statDirectory + untested truncate --- index.js | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index 45185723..48417407 100644 --- a/index.js +++ b/index.js @@ -438,6 +438,38 @@ class Hyperdrive extends EventEmitter { stream.end() } + truncate (name, size, cb) { + name = unixify(name) + + this.contentReady(err => { + if (err) return cb(err) + this._db.get(name, (err, st) => { + if (err) return cb(err) + if (!st) return this.writeFile(name, Buffer.alloc(size), cb) + try { + st = messages.Stat.decode(st.value) + } catch (err) { + return cb(err) + } + if (size === st.size) return cb(null) + if (size < st.size) { + const readStream = this.createReadStream(name, { length: size }) + const writeStream = this.createWriteStream(name) + return pump(readStream, writeStream, cb) + } else { + this.open(name, 'a', (err, fd) => { + if (err) return cb(err) + const length = size - st.size + this.write(fd, Buffer.alloc(length), 0, length, st.size, err => { + if (err) return cb(err) + this.close(fd, cb) + }) + }) + } + }) + }) + } + mkdir (name, opts, cb) { if (typeof opts === 'function') return this.mkdir(name, null, opts) if (typeof opts === 'number') opts = {mode: opts} @@ -473,12 +505,11 @@ class Hyperdrive extends EventEmitter { ite.next((err, st) => { if (err) return cb(err) if (name !== '/' && !st) return cb(new errors.FileNotFound(name)) - if (st) { - try { - st = messages.Stat.decode(st.value) - } catch (err) { - return cb(err) - } + if (name === '/') return cb(null, Stat.directory()) + try { + st = messages.Stat.decode(st.value) + } catch (err) { + return cb(err) } return cb(null, Stat.directory(st)) }) @@ -535,7 +566,6 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'function') return this.readdir(name, null, opts) name = unixify(name) - let dirStream = this.createDirectoryStream(name, opts) this._db.list(name, (err, list) => { if (err) return cb(err) return cb(null, list.map(st => name === '/' ? st.key : path.basename(name, st.key))) From d4b512803520c65172456f05fb013dbcb436cc86 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 21 Feb 2019 05:36:53 -0800 Subject: [PATCH 038/108] Handful of untested, late-night changes --- index.js | 10 +++++++++- lib/fd.js | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 48417407..46b40ad8 100644 --- a/index.js +++ b/index.js @@ -252,6 +252,7 @@ class Hyperdrive extends EventEmitter { const desc = this._fds[fd - STDIO_CAP] if (!desc) return process.nextTick(cb, new errors.BadFileDescriptor(`Bad file descriptor: ${fd}`)) if (pos == null) pos = desc.position + console.log('reading into buffer at buffer offset:', offset, len, 'bytes at pos', pos) desc.read(buf, offset, len, pos, cb) } @@ -441,16 +442,22 @@ class Hyperdrive extends EventEmitter { truncate (name, size, cb) { name = unixify(name) + this.ready(err => { + if (err) return cb(err) + this._update(name, { size: size }, cb) + }) + /* this.contentReady(err => { if (err) return cb(err) this._db.get(name, (err, st) => { if (err) return cb(err) - if (!st) return this.writeFile(name, Buffer.alloc(size), cb) + if (!st || !size) return this.writeFile(name, Buffer.alloc(size), cb) try { st = messages.Stat.decode(st.value) } catch (err) { return cb(err) } + console.log('truncating to size:', size, 'from size:', st.size) if (size === st.size) return cb(null) if (size < st.size) { const readStream = this.createReadStream(name, { length: size }) @@ -468,6 +475,7 @@ class Hyperdrive extends EventEmitter { } }) }) + */ } mkdir (name, opts, cb) { diff --git a/lib/fd.js b/lib/fd.js index 2819ad7b..2bb88ebe 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -36,11 +36,15 @@ class FileDescriptor { this._writeStream.on('error', err => { this._err = err }) - if (this.appending) this._appendStream = this.drive.createReadStream(this.path) + if (this.appending) { + this._appendStream = this.drive.createReadStream(this.path) + this.position = this.stat.size + } } } read (buffer, offset, len, pos, cb) { + console.log('READING', len, 'at', pos, 'with stat:', this.stat) if (!this.readable) return cb(new errors.BadFileDescriptor('File descriptor not open for reading.')) if (this.position !== null && this.position === pos) this._read(buffer, offset, len, cb) else this._seekAndRead(buffer, offset, len, pos, cb) @@ -55,6 +59,7 @@ class FileDescriptor { return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) } if (this.appending && pos < this.stat.size) { + console.log('pos:', pos, 'this.stat.size:', this.stat.size) return cb(new errors.BadFileDescriptor('Position cannot be less than the file size when appending.')) } if (!this._writeStream && !this.appending && pos !== 0) { @@ -142,6 +147,7 @@ class FileDescriptor { if (err) return cb(err) if (blkOffset) data = data.slice(blkOffset) + console.log('copying data:', data, 'into buf') data.copy(buf) const read = Math.min(data.length, buf.length) @@ -174,6 +180,8 @@ FileDescriptor.create = function (drive, name, flags, cb) { const creating = !!(flags & O_CREAT) const canExist = !(flags & O_EXCL) + console.log(`FD FOR ${name} append? ${appending} readable? ${readable} writeable? ${writable}`) + drive.contentReady(err => { if (err) return cb(err) drive._db.get(name, (err, st) => { From 8bc8ca387adfb10db4717a03ad2a3caeeb11007b Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 22 Feb 2019 00:45:09 -0800 Subject: [PATCH 039/108] Many bug fixes + create --- index.js | 44 +++++++++++++++++++++++++++++--------------- lib/fd.js | 42 +++++++++++++++++++++++++++++++----------- lib/stat.js | 6 +++--- package.json | 2 ++ 4 files changed, 65 insertions(+), 29 deletions(-) diff --git a/index.js b/index.js index 46b40ad8..00d5791a 100644 --- a/index.js +++ b/index.js @@ -252,7 +252,6 @@ class Hyperdrive extends EventEmitter { const desc = this._fds[fd - STDIO_CAP] if (!desc) return process.nextTick(cb, new errors.BadFileDescriptor(`Bad file descriptor: ${fd}`)) if (pos == null) pos = desc.position - console.log('reading into buffer at buffer offset:', offset, len, 'bytes at pos', pos) desc.read(buf, offset, len, pos, cb) } @@ -379,11 +378,12 @@ class Hyperdrive extends EventEmitter { proxy.on('prefinish', function () { const stat = Stat.file({ ...opts, - offset: offset, - byteOffset: byteOffset, + offset, + byteOffset, size: self.contentFeed.byteLength - byteOffset, blocks: self.contentFeed.length - offset }) + try { var encoded = messages.Stat.encode(stat) } catch (err) { @@ -408,6 +408,25 @@ class Hyperdrive extends EventEmitter { } } + create (name, opts, cb) { + if (typeof opts === 'function') return this.create(name, null, opts) + + name = unixify(name) + + this.ready(err => { + if (err) return cb(err) + try { + var st = messages.Stat.encode(Stat.file(opts)) + } catch (err) { + return cb(err) + } + this._db.put(name, st, err => { + if (err) return cb(err) + return cb(null) + }) + }) + } + readFile (name, opts, cb) { if (typeof opts === 'function') return this.readFile(name, null, opts) if (typeof opts === 'string') opts = {encoding: opts} @@ -431,22 +450,15 @@ class Hyperdrive extends EventEmitter { name = unixify(name) - let bufs = split(buf) // split the input incase it is a big buffer. let stream = this.createWriteStream(name, opts) stream.on('error', cb) stream.on('finish', cb) - for (let i = 0; i < bufs.length; i++) stream.write(bufs[i]) - stream.end() + stream.end(buf) } truncate (name, size, cb) { name = unixify(name) - this.ready(err => { - if (err) return cb(err) - this._update(name, { size: size }, cb) - }) - /* this.contentReady(err => { if (err) return cb(err) this._db.get(name, (err, st) => { @@ -457,7 +469,6 @@ class Hyperdrive extends EventEmitter { } catch (err) { return cb(err) } - console.log('truncating to size:', size, 'from size:', st.size) if (size === st.size) return cb(null) if (size < st.size) { const readStream = this.createReadStream(name, { length: size }) @@ -475,7 +486,6 @@ class Hyperdrive extends EventEmitter { } }) }) - */ } mkdir (name, opts, cb) { @@ -572,11 +582,15 @@ class Hyperdrive extends EventEmitter { readdir (name, opts, cb) { if (typeof opts === 'function') return this.readdir(name, null, opts) + name = unixify(name) + if (name !== '/' && name.startsWith('/')) name = name.slice(1) + + const recursive = !!(opts && opts.recursive) - this._db.list(name, (err, list) => { + this._db.list(name, { gt: true, recursive }, (err, list) => { if (err) return cb(err) - return cb(null, list.map(st => name === '/' ? st.key : path.basename(name, st.key))) + return cb(null, list.map(st => name === '/' ? st.key.split('/')[0] : path.basename(st.key, name))) }) } diff --git a/lib/fd.js b/lib/fd.js index 2bb88ebe..b1a68f48 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -32,10 +32,6 @@ class FileDescriptor { this._err = null if (this.writable) { - this._writeStream = this.drive.createWriteStream(this.path) - this._writeStream.on('error', err => { - this._err = err - }) if (this.appending) { this._appendStream = this.drive.createReadStream(this.path) this.position = this.stat.size @@ -44,7 +40,6 @@ class FileDescriptor { } read (buffer, offset, len, pos, cb) { - console.log('READING', len, 'at', pos, 'with stat:', this.stat) if (!this.readable) return cb(new errors.BadFileDescriptor('File descriptor not open for reading.')) if (this.position !== null && this.position === pos) this._read(buffer, offset, len, cb) else this._seekAndRead(buffer, offset, len, pos, cb) @@ -59,14 +54,19 @@ class FileDescriptor { return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) } if (this.appending && pos < this.stat.size) { - console.log('pos:', pos, 'this.stat.size:', this.stat.size) return cb(new errors.BadFileDescriptor('Position cannot be less than the file size when appending.')) } - if (!this._writeStream && !this.appending && pos !== 0) { + if (!this._writeStream && !this.appending && pos) { return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) } const self = this + if (!this._writeStream) { + this._writeStream = this.drive.createWriteStream(this.path) + this._writeStream.on('error', err => { + this._err = err + }) + } // TODO: This is a temporary (bad) way of supporting appends. if (this._appendStream) { @@ -135,6 +135,29 @@ class FileDescriptor { } _read (buffer, offset, len, cb) { + const self = this + + let totalRead = 0 + let available = len + readNextBlock() + + function readNextBlock () { + self._readBlock(buffer, offset + totalRead, available, (err, bytesRead) => { + if (err) return cb(err) + if (!bytesRead) return cb(null, totalRead, buffer) + + totalRead += bytesRead + available -= bytesRead + + if (totalRead < 4096 && available) { + return readNextBlock() + } + return cb(null, totalRead, buffer) + }) + } + } + + _readBlock (buffer, offset, len, cb) { const buf = buffer.slice(offset, offset + len) const blkOffset = this.blockOffset const blk = this.blockPosition @@ -147,7 +170,6 @@ class FileDescriptor { if (err) return cb(err) if (blkOffset) data = data.slice(blkOffset) - console.log('copying data:', data, 'into buf') data.copy(buf) const read = Math.min(data.length, buf.length) @@ -180,8 +202,6 @@ FileDescriptor.create = function (drive, name, flags, cb) { const creating = !!(flags & O_CREAT) const canExist = !(flags & O_EXCL) - console.log(`FD FOR ${name} append? ${appending} readable? ${readable} writeable? ${writable}`) - drive.contentReady(err => { if (err) return cb(err) drive._db.get(name, (err, st) => { @@ -197,7 +217,7 @@ FileDescriptor.create = function (drive, name, flags, cb) { } } - cb(null, new FileDescriptor(drive, name, st, readable, writable, appending, creating)) + return cb(null, new FileDescriptor(drive, name, st, readable, writable, appending, creating)) }) }) } diff --git a/lib/stat.js b/lib/stat.js index 875467b8..873c59ee 100644 --- a/lib/stat.js +++ b/lib/stat.js @@ -17,9 +17,9 @@ class Stat { this.offset = (data && data.offset) || 0 this.byteOffset = (data && data.byteOffset) || 0 this.blocks = (data && data.blocks) || 0 - this.atime = data && data.atime ? getTime(data.atime) : 0 // we just set this to mtime ... - this.mtime = data && data.mtime ? getTime(data.mtime) : 0 - this.ctime = data && data.ctime? getTime(data.ctime) : 0 + this.atime = data && data.atime ? getTime(data.atime) : Date.now() // we just set this to mtime ... + this.mtime = data && data.mtime ? getTime(data.mtime) : Date.now() + this.ctime = data && data.ctime ? getTime(data.ctime) : Date.now() this.linkname = (data && data.linkname) || null } diff --git a/package.json b/package.json index 8c03b487..2a4de273 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ }, "homepage": "https://github.com/mafintosh/hyperdrive#readme", "dependencies": { + "byte-stream": "^2.1.0", "custom-error-class": "^1.0.0", "duplexify": "^3.7.1", "hypercore": "^6.25.0", @@ -28,6 +29,7 @@ "hypertrie": "^3.4.0", "mutexify": "^1.2.0", "pump": "^3.0.0", + "pumpify": "^1.5.1", "sodium-universal": "^2.0.0", "stream-collector": "^1.0.1", "through2": "^3.0.0", From 4191466f034dff72de1476d6ccd6b8df70d55330 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 22 Feb 2019 10:33:12 -0800 Subject: [PATCH 040/108] Temporarily add debugging --- index.js | 2 +- lib/fd.js | 27 ++++++++++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 00d5791a..032b8b58 100644 --- a/index.js +++ b/index.js @@ -463,7 +463,7 @@ class Hyperdrive extends EventEmitter { if (err) return cb(err) this._db.get(name, (err, st) => { if (err) return cb(err) - if (!st || !size) return this.writeFile(name, Buffer.alloc(size), cb) + if (!st || !size) return this.create(name, cb) try { st = messages.Stat.decode(st.value) } catch (err) { diff --git a/lib/fd.js b/lib/fd.js index b1a68f48..11157211 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -64,12 +64,12 @@ class FileDescriptor { if (!this._writeStream) { this._writeStream = this.drive.createWriteStream(this.path) this._writeStream.on('error', err => { - this._err = err + return cb(err) }) } // TODO: This is a temporary (bad) way of supporting appends. - if (this._appendStream) { + if (this._appendStream && false) { this.position = this.stat.size // pump does not support the `end` option. this._appendStream.pipe(this._writeStream, { end: false }) @@ -86,9 +86,11 @@ class FileDescriptor { if (self._err) return cb(self._err) if (self._writeStream.destroyed) return cb(new errors.BadFileDescriptor('Write stream was destroyed.')) + self._appendStream = null if (err) return cb(err) const slice = buffer.slice(offset, len) + console.log('WRITING', len, 'at pos:', pos, 'offset:', offset, 'slice:', slice, 'feed length:', self.drive.contentFeed.length) write(self._writeStream, slice, err => { if (err) return cb(err) self.position += slice.length @@ -130,11 +132,13 @@ class FileDescriptor { this.position = pos this.blockPosition = blk this.blockOffset = blockOffset + console.log('this.stat:', this.stat, 'seeked to:', pos, 'start:', start, 'end:', end) this._read(buffer, offset, len, cb) }) } _read (buffer, offset, len, cb) { + console.log('at beginning of read, stat:', this.stat) const self = this let totalRead = 0 @@ -142,16 +146,17 @@ class FileDescriptor { readNextBlock() function readNextBlock () { - self._readBlock(buffer, offset + totalRead, available, (err, bytesRead) => { + self._readBlock(buffer, offset + totalRead, Math.max(available, 0), (err, bytesRead) => { if (err) return cb(err) if (!bytesRead) return cb(null, totalRead, buffer) totalRead += bytesRead available -= bytesRead - if (totalRead < 4096 && available) { + if (available > 0) { return readNextBlock() } + console.log('AT END OF READ, buffer:', buffer) return cb(null, totalRead, buffer) }) } @@ -161,6 +166,7 @@ class FileDescriptor { const buf = buffer.slice(offset, offset + len) const blkOffset = this.blockOffset const blk = this.blockPosition + console.log('reading block at position:', blk) if ((this.stat.offset + this.stat.blocks) <= blk || blk < this.stat.offset) { return process.nextTick(cb, null, 0, buffer) @@ -169,6 +175,7 @@ class FileDescriptor { this.drive.contentFeed.get(blk, (err, data) => { if (err) return cb(err) if (blkOffset) data = data.slice(blkOffset) + console.log('blkOffset:', blkOffset, 'blk:', blk, 'data:', data) data.copy(buf) const read = Math.min(data.length, buf.length) @@ -202,6 +209,8 @@ FileDescriptor.create = function (drive, name, flags, cb) { const creating = !!(flags & O_CREAT) const canExist = !(flags & O_EXCL) + console.log('readable:', readable, 'writable:', writable, 'appending:', appending, 'creating:', creating) + drive.contentReady(err => { if (err) return cb(err) drive._db.get(name, (err, st) => { @@ -217,7 +226,15 @@ FileDescriptor.create = function (drive, name, flags, cb) { } } - return cb(null, new FileDescriptor(drive, name, st, readable, writable, appending, creating)) + const fd = new FileDescriptor(drive, name, st, readable, writable, appending, creating) + if (creating) { + drive.create(name, err => { + if (err) return cb(err) + return cb(null, fd) + }) + } else { + return cb(null, fd) + } }) }) } From c633824dc832e6d588a41f9bacf6610e4e4d3df3 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 23 Feb 2019 18:30:43 -0800 Subject: [PATCH 041/108] Temporary buffer copy in writable fd (fuse workaround) --- lib/fd.js | 17 +++++----------- test/fd.js | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/lib/fd.js b/lib/fd.js index 11157211..0e77b8e1 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -60,6 +60,8 @@ class FileDescriptor { return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) } + buffer = Buffer.from(buffer) + const self = this if (!this._writeStream) { this._writeStream = this.drive.createWriteStream(this.path) @@ -69,7 +71,7 @@ class FileDescriptor { } // TODO: This is a temporary (bad) way of supporting appends. - if (this._appendStream && false) { + if (this._appendStream) { this.position = this.stat.size // pump does not support the `end` option. this._appendStream.pipe(this._writeStream, { end: false }) @@ -83,14 +85,12 @@ class FileDescriptor { return doWrite() function doWrite (err) { + self._appendStream = null + if (err) return cb(err) if (self._err) return cb(self._err) if (self._writeStream.destroyed) return cb(new errors.BadFileDescriptor('Write stream was destroyed.')) - - self._appendStream = null - if (err) return cb(err) const slice = buffer.slice(offset, len) - console.log('WRITING', len, 'at pos:', pos, 'offset:', offset, 'slice:', slice, 'feed length:', self.drive.contentFeed.length) write(self._writeStream, slice, err => { if (err) return cb(err) self.position += slice.length @@ -132,13 +132,11 @@ class FileDescriptor { this.position = pos this.blockPosition = blk this.blockOffset = blockOffset - console.log('this.stat:', this.stat, 'seeked to:', pos, 'start:', start, 'end:', end) this._read(buffer, offset, len, cb) }) } _read (buffer, offset, len, cb) { - console.log('at beginning of read, stat:', this.stat) const self = this let totalRead = 0 @@ -156,7 +154,6 @@ class FileDescriptor { if (available > 0) { return readNextBlock() } - console.log('AT END OF READ, buffer:', buffer) return cb(null, totalRead, buffer) }) } @@ -166,7 +163,6 @@ class FileDescriptor { const buf = buffer.slice(offset, offset + len) const blkOffset = this.blockOffset const blk = this.blockPosition - console.log('reading block at position:', blk) if ((this.stat.offset + this.stat.blocks) <= blk || blk < this.stat.offset) { return process.nextTick(cb, null, 0, buffer) @@ -175,7 +171,6 @@ class FileDescriptor { this.drive.contentFeed.get(blk, (err, data) => { if (err) return cb(err) if (blkOffset) data = data.slice(blkOffset) - console.log('blkOffset:', blkOffset, 'blk:', blk, 'data:', data) data.copy(buf) const read = Math.min(data.length, buf.length) @@ -209,8 +204,6 @@ FileDescriptor.create = function (drive, name, flags, cb) { const creating = !!(flags & O_CREAT) const canExist = !(flags & O_EXCL) - console.log('readable:', readable, 'writable:', writable, 'appending:', appending, 'creating:', creating) - drive.contentReady(err => { if (err) return cb(err) drive._db.get(name, (err, st) => { diff --git a/test/fd.js b/test/fd.js index ff9070a1..b76c0856 100644 --- a/test/fd.js +++ b/test/fd.js @@ -1,5 +1,6 @@ const tape = require('tape') const create = require('./helpers/create') +const hyperdrive = require('..') tape('basic fd read', function (t) { const drive = create() @@ -282,6 +283,63 @@ tape('fd stateful write', function (t) { }) }) +tape('huge stateful write + stateless read', function (t) { + const NUM_SLICES = 1000 + const SLICE_SIZE = 4096 + const READ_SIZE = Math.floor(4096 * 2.75) + + const drive = create() + + const content = Buffer.alloc(SLICE_SIZE * NUM_SLICES).fill('0123456789abcdefghijklmnopqrstuvwxyz') + let slices = new Array(NUM_SLICES).fill(0).map((_, i) => content.slice(SLICE_SIZE * i, SLICE_SIZE * (i+1))) + + drive.open('hello', 'w+', function (err, fd) { + t.error(err, 'no error') + writeSlices(drive, fd, err => { + t.error(err, 'no errors during write') + drive.open('hello', 'r', function (err, fd) { + t.error(err, 'no error') + compareSlices(drive, fd) + }) + }) + }) + + function compareSlices (drive, fd) { + let read = 0 + readNext() + + function readNext () { + const buf = Buffer.alloc(READ_SIZE) + const pos = read * READ_SIZE + drive.read(fd, buf, 0, READ_SIZE, pos, (err, bytesRead) => { + if (err) return t.fail(err) + if (!buf.slice(0, bytesRead).equals(content.slice(pos, pos + READ_SIZE))) { + return t.fail(`Slices do not match at position: ${read}`) + } + if (++read * READ_SIZE >= NUM_SLICES * SLICE_SIZE) { + console.log('they all matched') + return t.end() + } + return readNext(drive, fd) + }) + } + } + + function writeSlices (drive, fd, cb) { + let written = 0 + writeNext() + + function writeNext () { + const buf = slices[written] + drive.write(fd, buf, 0, SLICE_SIZE, err => { + if (err) return cb(err) + if (++written === NUM_SLICES) return drive.close(fd, cb) + return writeNext() + }) + } + } +}) + tape('fd random-access write fails', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') From 0c5db686585d01c4a052c684003fb86b85fdbb4a Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 23 Feb 2019 22:43:55 -0800 Subject: [PATCH 042/108] Added batcher for fewer blocks --- lib/fd.js | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/lib/fd.js b/lib/fd.js index 0e77b8e1..cfd9d688 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -1,8 +1,13 @@ const fs = require('fs') const pump = require('pump') const errors = require('./errors') +const byteStream = require('byte-stream') +const through = require('through2') +const pumpify = require('pumpify') const messages = require('./messages') +const noop = () => {} + const { O_RDONLY, O_WRONLY, @@ -37,6 +42,10 @@ class FileDescriptor { this.position = this.stat.size } } + + this._batcher = byteStream({ time: 500, limit: 4096 * 100 }) + + this._range = null } read (buffer, offset, len, pos, cb) { @@ -64,7 +73,11 @@ class FileDescriptor { const self = this if (!this._writeStream) { - this._writeStream = this.drive.createWriteStream(this.path) + const writeStream = this.drive.createWriteStream(this.path) + this._writeStream = pumpify(this._batcher, through.obj((chunk, enc, cb) => { + console.log('chunk:', chunk) + cb(null, Buffer.concat(chunk)) + }), writeStream) this._writeStream.on('error', err => { return cb(err) }) @@ -91,7 +104,9 @@ class FileDescriptor { if (self._writeStream.destroyed) return cb(new errors.BadFileDescriptor('Write stream was destroyed.')) const slice = buffer.slice(offset, len) + console.log('writing slice:', slice, 'to batcher') write(self._writeStream, slice, err => { + console.log('err:', err) if (err) return cb(err) self.position += slice.length return cb(null, slice.length, buffer) @@ -120,18 +135,37 @@ class FileDescriptor { }) } } + if (this._range) { + this.drive.contentFeed.undownload(this._range) + this._range = null + } process.nextTick(cb, null) } + _refreshDownload (start, cb) { + let end = Math.min(this.stat.blocks + this.stat.offset, start + 500) + + if (this._range) { + console.log('UNDOWNLOADING:', this._range) + this.drive.contentFeed.undownload(this._range) + } + + console.log(`DOWNLOADING FROM ${start} TO ${end} randomly`) + this._range = this.drive.contentFeed.download({ start, end, linear: true }, cb || noop) + } + _seekAndRead (buffer, offset, len, pos, cb) { const start = this.stat.offset const end = start + this.stat.blocks + this.drive.contentFeed.seek(this.stat.byteOffset + pos, { start, end }, (err, blk, blockOffset) => { if (err) return cb(err) this.position = pos this.blockPosition = blk this.blockOffset = blockOffset + + this._refreshDownload(blk) this._read(buffer, offset, len, cb) }) } @@ -154,6 +188,7 @@ class FileDescriptor { if (available > 0) { return readNextBlock() } + console.log('totalRead:', totalRead, 'len:', len) return cb(null, totalRead, buffer) }) } @@ -164,11 +199,17 @@ class FileDescriptor { const blkOffset = this.blockOffset const blk = this.blockPosition + if (this._range && (blk < this._range.start || blk > this._range.end)) { + this._refreshDownload(blk) + } + if ((this.stat.offset + this.stat.blocks) <= blk || blk < this.stat.offset) { return process.nextTick(cb, null, 0, buffer) } + console.log('GETTING BLOCK:', blk) this.drive.contentFeed.get(blk, (err, data) => { + console.log('GOT BLOCK:', blk) if (err) return cb(err) if (blkOffset) data = data.slice(blkOffset) From 7aeb1249ddcc2653ee7a20ef898b4156d0ee635e Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sun, 24 Feb 2019 01:20:41 -0800 Subject: [PATCH 043/108] Prefetch limit + readdir bug --- index.js | 5 ++++- lib/fd.js | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 032b8b58..e21483a4 100644 --- a/index.js +++ b/index.js @@ -590,7 +590,10 @@ class Hyperdrive extends EventEmitter { this._db.list(name, { gt: true, recursive }, (err, list) => { if (err) return cb(err) - return cb(null, list.map(st => name === '/' ? st.key.split('/')[0] : path.basename(st.key, name))) + return cb(null, list.map(st => { + if (name === '/') return st.key.split('/')[0] + return path.relative(name, st.key).split('/')[0] + })) }) } diff --git a/lib/fd.js b/lib/fd.js index cfd9d688..35536eee 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -143,14 +143,14 @@ class FileDescriptor { } _refreshDownload (start, cb) { - let end = Math.min(this.stat.blocks + this.stat.offset, start + 500) + let end = Math.min(this.stat.blocks + this.stat.offset, start + 15) if (this._range) { console.log('UNDOWNLOADING:', this._range) this.drive.contentFeed.undownload(this._range) } - console.log(`DOWNLOADING FROM ${start} TO ${end} randomly`) + console.log(`DOWNLOADING FROM ${start} TO ${end} linearly`) this._range = this.drive.contentFeed.download({ start, end, linear: true }, cb || noop) } From 9fbe03d957780925651415bd305a86b51750a9de Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sun, 24 Feb 2019 15:29:30 -0800 Subject: [PATCH 044/108] Nested directory readdir test --- lib/fd.js | 10 +--------- test/basic.js | 46 ++++++++++++++++++++++++++++++++++++++++++++++ test/fd.js | 1 - 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/lib/fd.js b/lib/fd.js index 35536eee..43321a04 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -43,7 +43,7 @@ class FileDescriptor { } } - this._batcher = byteStream({ time: 500, limit: 4096 * 100 }) + this._batcher = byteStream({ time: 100, limit: 4096 * 16 }) this._range = null } @@ -75,7 +75,6 @@ class FileDescriptor { if (!this._writeStream) { const writeStream = this.drive.createWriteStream(this.path) this._writeStream = pumpify(this._batcher, through.obj((chunk, enc, cb) => { - console.log('chunk:', chunk) cb(null, Buffer.concat(chunk)) }), writeStream) this._writeStream.on('error', err => { @@ -104,9 +103,7 @@ class FileDescriptor { if (self._writeStream.destroyed) return cb(new errors.BadFileDescriptor('Write stream was destroyed.')) const slice = buffer.slice(offset, len) - console.log('writing slice:', slice, 'to batcher') write(self._writeStream, slice, err => { - console.log('err:', err) if (err) return cb(err) self.position += slice.length return cb(null, slice.length, buffer) @@ -146,11 +143,9 @@ class FileDescriptor { let end = Math.min(this.stat.blocks + this.stat.offset, start + 15) if (this._range) { - console.log('UNDOWNLOADING:', this._range) this.drive.contentFeed.undownload(this._range) } - console.log(`DOWNLOADING FROM ${start} TO ${end} linearly`) this._range = this.drive.contentFeed.download({ start, end, linear: true }, cb || noop) } @@ -188,7 +183,6 @@ class FileDescriptor { if (available > 0) { return readNextBlock() } - console.log('totalRead:', totalRead, 'len:', len) return cb(null, totalRead, buffer) }) } @@ -207,9 +201,7 @@ class FileDescriptor { return process.nextTick(cb, null, 0, buffer) } - console.log('GETTING BLOCK:', blk) this.drive.contentFeed.get(blk, (err, data) => { - console.log('GOT BLOCK:', blk) if (err) return cb(err) if (blkOffset) data = data.slice(blkOffset) diff --git a/test/basic.js b/test/basic.js index 86dde142..4a452667 100644 --- a/test/basic.js +++ b/test/basic.js @@ -316,3 +316,49 @@ tape('can stream a large directory', async function (t) { }) } }) + +tape('can read nested directories', async function (t) { + const drive = create(null) + + let files = ['a', 'b/a/b', 'b/c', 'c/b', 'd/e/f/g/h', 'd/e/a', 'e/a', 'e/b', 'f', 'g'] + let rootSet = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g']) + let bSet = new Set(['a', 'c']) + let dSet = new Set(['e']) + let eSet = new Set(['a', 'b']) + let deSet = new Set(['f', 'a']) + + for (let file of files) { + await insertFile(file, 'a small file') + } + + await checkDir('/', rootSet) + await checkDir('b', bSet) + await checkDir('d', dSet) + await checkDir('e', eSet) + await checkDir('d/e', deSet) + + t.end() + + function checkDir (dir, fileSet) { + return new Promise(resolve => { + drive.readdir(dir, (err, files) => { + t.error(err, 'no error') + for (let file of files) { + t.true(fileSet.has(file), 'correct file was listed') + fileSet.delete(file) + } + t.same(fileSet.size, 0, 'all files were listed') + return resolve() + }) + }) + } + + function insertFile (name, content) { + return new Promise((resolve, reject) => { + drive.writeFile(name, content, err => { + if (err) return reject(err) + return resolve() + }) + }) + } +}) diff --git a/test/fd.js b/test/fd.js index b76c0856..b7fefd6c 100644 --- a/test/fd.js +++ b/test/fd.js @@ -317,7 +317,6 @@ tape('huge stateful write + stateless read', function (t) { return t.fail(`Slices do not match at position: ${read}`) } if (++read * READ_SIZE >= NUM_SLICES * SLICE_SIZE) { - console.log('they all matched') return t.end() } return readNext(drive, fd) From 3b83a53d3d7fc05774c151ed7c4b812031e4ea8a Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sun, 24 Feb 2019 15:44:57 -0800 Subject: [PATCH 045/108] Added sparseMetadata test --- index.js | 9 ++++++++- test/basic.js | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index e21483a4..9565d585 100644 --- a/index.js +++ b/index.js @@ -40,13 +40,14 @@ class Hyperdrive extends EventEmitter { this.live = true this.latest = !!opts.latest this.sparse = !!opts.sparse + this.sparseMetadata = !!opts.sparseMetadata this._factory = opts.factory ? storage : null this._storages = !this.factory ? defaultStorage(this, storage, opts) : null this.metadataFeed = opts.metadataFeed || this._createHypercore(this._storages.metadata, key, { secretKey: opts.secretKey, - sparse: !!opts.sparseMetadata, + sparse: this.sparseMetadata, createIfMissing: opts.createIfMissing, storageCacheSize: opts.metadataStorageCacheSize, valueEncoding: 'binary' @@ -104,6 +105,12 @@ class Hyperdrive extends EventEmitter { this.metadataFeed.ready(err => { if (err) return cb(err) + if (this.sparseMetadata) { + this.metadataFeed.update(function loop () { + self.metadataFeed.update(loop) + }) + } + this._contentKeyPair = this.metadataFeed.secretKey ? contentKeyPair(this.metadataFeed.secretKey) : {} this._contentOpts = contentOptions(this, this._contentKeyPair.secretKey) this._contentOpts.keyPair = this._contentKeyPair diff --git a/test/basic.js b/test/basic.js index 4a452667..97f76685 100644 --- a/test/basic.js +++ b/test/basic.js @@ -362,3 +362,47 @@ tape('can read nested directories', async function (t) { }) } }) + +tape('can read sparse metadata', async function (t) { + const { read, write } = await getTestDrives() + + let files = ['a', 'b/a/b', 'b/c', 'c/b', 'd/e/f/g/h', 'd/e/a', 'e/a', 'e/b', 'f', 'g'] + + for (let file of files) { + await insertFile(file, 'a small file') + await checkFile(file) + } + + t.end() + + function checkFile (file) { + return new Promise(resolve => { + read.stat(file, (err, st) => { + t.error(err, 'no error') + t.true(st) + return resolve() + }) + }) + } + + function insertFile (name, content) { + return new Promise((resolve, reject) => { + write.writeFile(name, content, err => { + if (err) return reject(err) + return resolve() + }) + }) + } + + function getTestDrives () { + return new Promise(resolve => { + let drive = create() + drive.on('ready', () => { + let clone = create(drive.key, { sparseMetadata: true, sparse: true }) + let s1 = clone.replicate({ live: true }) + s1.pipe(drive.replicate({ live: true })).pipe(s1) + return resolve({ read: clone, write: drive }) + }) + }) + } +}) From d1b650241848e0c329c9ed6ee48edf419c5aa426 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sun, 24 Feb 2019 17:00:31 -0800 Subject: [PATCH 046/108] Prefetch 16 blocks in fd + add comment --- lib/fd.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/fd.js b/lib/fd.js index 43321a04..2af4ca39 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -69,8 +69,6 @@ class FileDescriptor { return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) } - buffer = Buffer.from(buffer) - const self = this if (!this._writeStream) { const writeStream = this.drive.createWriteStream(this.path) @@ -139,8 +137,13 @@ class FileDescriptor { process.nextTick(cb, null) } + /** + * Will currently request the next 16 blocks linearly. + * + * TODO: This behavior should be more customizable in the future. + */ _refreshDownload (start, cb) { - let end = Math.min(this.stat.blocks + this.stat.offset, start + 15) + let end = Math.min(this.stat.blocks + this.stat.offset, start + 16) if (this._range) { this.drive.contentFeed.undownload(this._range) From afbb6878bdc8d01b93fe85f6bc0abe36a8bc4f31 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 25 Feb 2019 03:02:31 -0800 Subject: [PATCH 047/108] Add createDiffStream (do not use until trie bug fix) --- index.js | 29 +++++++++++++++++++++++++ test/basic.js | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/index.js b/index.js index 9565d585..4150365c 100644 --- a/index.js +++ b/index.js @@ -7,6 +7,7 @@ const unixify = require('unixify') const mutexify = require('mutexify') const duplexify = require('duplexify') const through = require('through2') +const pumpify = require('pumpify') const pump = require('pump') const hypercore = require('hypercore') @@ -314,6 +315,34 @@ class Hyperdrive extends EventEmitter { return stream } + createDiffStream (other, prefix, opts) { + if (other instanceof Hyperdrive) other = other.version + if (typeof prefix === 'object') return this.createDiffStream(other, '/', prefix) + prefix = prefix || '/' + + const proxy = duplexify.obj() + proxy.setWritable(false) + + this.ready(err => { + if (err) return proxy.destroy(err) + + const decoder = through.obj((chunk, enc, cb) => { + let obj = { type: !chunk.left ? 'del' : 'put', name: chunk.key} + if (chunk.left) { + try { + obj.stat = messages.Stat.decode(chunk.left.value) + } catch (err) { + return cb(err) + } + } + return cb(null, obj) + }) + proxy.setReadable(pumpify.obj(this._db.createDiffStream(other, prefix, opts), decoder)) + }) + + return proxy + } + createDirectoryStream (name, opts) { if (!opts) opts = {} diff --git a/test/basic.js b/test/basic.js index 97f76685..2b977d88 100644 --- a/test/basic.js +++ b/test/basic.js @@ -406,3 +406,62 @@ tape('can read sparse metadata', async function (t) { }) } }) + +// TODO: Revisit createDiffStream after hypertrie diff stream bug is fixed. +/* +tape.only('simple diff stream', async function (t) { + let drive = create() + + var v1, v2, v3 + let v3Diff = ['del-hello'] + let v2Diff = [...v3Diff, 'put-other'] + let v1Diff = [...v2Diff, 'put-hello'] + + await writeVersions() + console.log('drive.version:', drive.version, 'v1:', v1) + // await verifyDiffStream(v1, v1Diff) + // await verifyDiffStream(v2, v2Diff) + await verifyDiffStream(v3, v3Diff) + t.end() + + function writeVersions () { + return new Promise(resolve => { + drive.ready(err => { + t.error(err, 'no error') + v1 = drive.version + drive.writeFile('/hello', 'world', err => { + t.error(err, 'no error') + v2 = drive.version + drive.writeFile('/other', 'file', err => { + t.error(err, 'no error') + v3 = drive.version + drive.unlink('/hello', err => { + t.error(err, 'no error') + return resolve() + }) + }) + }) + }) + }) + } + + async function verifyDiffStream (version, diffList) { + let diffSet = new Set(diffList) + console.log('diffing to version:', version, 'from version:', drive.version) + let diffStream = drive.createDiffStream(version) + return new Promise(resolve => { + diffStream.on('data', ({ type, name }) => { + let key = `${type}-${name}` + if (!diffSet.has(key)) { + return t.fail('an incorrect diff was streamed') + } + diffSet.delete(key) + }) + diffStream.on('end', () => { + t.same(diffSet.size, 0) + return resolve() + }) + }) + } +}) +*/ From 81dbc090f253ae5f48160c1c55bb3d329abc42f5 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 30 Mar 2019 20:51:18 -0700 Subject: [PATCH 048/108] Assorted changes for @mafintosh's review --- README.md | 14 +++++++++ index.js | 82 ++++++++++++++++++++++++------------------------ lib/content.js | 2 +- lib/fd.js | 47 +++++++++++++-------------- lib/storage.js | 6 +--- test/basic.js | 8 ++--- test/creation.js | 4 +-- test/storage.js | 18 +++++------ 8 files changed, 96 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index feeaa6b8..793bd0ae 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,20 @@ Options include: If `wait` is set to `true`, this function will wait for data to be downloaded. If false, will return an error. +#### `archive.open(name, flags, callback)` + +Open a file and get a file descriptor back. Similar to fs.open. + +Note that currently only read mode is supported in this API. + +#### `archive.read(fd, buf, offset, len, position, callback)` + +Read from a file descriptor into a buffer. Similar to fs.read. + +#### `archive.write(fd, buf, offset, len, pos, cb)` + +Write from a buffer into a file descriptor. Similar to fs.write. + #### `archive.close(fd, [callback])` Close a file. Similar to fs.close. diff --git a/index.js b/index.js index 4150365c..413a9f5f 100644 --- a/index.js +++ b/index.js @@ -14,11 +14,11 @@ const hypercore = require('hypercore') const hypertrie = require('hypertrie') const coreByteStream = require('hypercore-byte-stream') -const FD = require('./lib/fd') +const createFileDescriptor = require('./lib/fd') const Stat = require('./lib/stat') const errors = require('./lib/errors') const messages = require('./lib/messages') -const { defaultStorage } = require('./lib/storage') +const defaultStorage = require('./lib/storage') const { contentKeyPair, contentOptions } = require('./lib/content') // 20 is arbitrary, just to make the fds > stdio etc @@ -46,7 +46,7 @@ class Hyperdrive extends EventEmitter { this._factory = opts.factory ? storage : null this._storages = !this.factory ? defaultStorage(this, storage, opts) : null - this.metadataFeed = opts.metadataFeed || this._createHypercore(this._storages.metadata, key, { + this.metadata = opts.metadata || this._createHypercore(this._storages.metadata, key, { secretKey: opts.secretKey, sparse: this.sparseMetadata, createIfMissing: opts.createIfMissing, @@ -54,7 +54,7 @@ class Hyperdrive extends EventEmitter { valueEncoding: 'binary' }) this._db = opts._db - this.contentFeed = opts.contentFeed || null + this.content = opts.content || null this.storage = storage this.contentStorageCacheSize = opts.contentStorageCacheSize @@ -94,25 +94,25 @@ class Hyperdrive extends EventEmitter { } get writable () { - return this.metadataFeed.writable && this.contentFeed.writable + return this.metadata.writable && this.content.writable } _ready (cb) { const self = this - this.metadataFeed.on('error', onerror) - this.metadataFeed.on('append', update) + this.metadata.on('error', onerror) + this.metadata.on('append', update) - this.metadataFeed.ready(err => { + this.metadata.ready(err => { if (err) return cb(err) if (this.sparseMetadata) { - this.metadataFeed.update(function loop () { - self.metadataFeed.update(loop) + this.metadata.update(function loop () { + self.metadata.update(loop) }) } - this._contentKeyPair = this.metadataFeed.secretKey ? contentKeyPair(this.metadataFeed.secretKey) : {} + this._contentKeyPair = this.metadata.secretKey ? contentKeyPair(this.metadata.secretKey) : {} this._contentOpts = contentOptions(this, this._contentKeyPair.secretKey) this._contentOpts.keyPair = this._contentKeyPair @@ -125,9 +125,9 @@ class Hyperdrive extends EventEmitter { * Initialize the db without metadata and load the content feed key from the header. */ if (this._db) { - if (!this.contentFeed || !this.metadataFeed) return cb(new Error('Must provide a db and both content/metadata feeds')) + if (!this.content || !this.metadata) return cb(new Error('Must provide a db and both content/metadata feeds')) return done(null) - } else if (this.metadataFeed.writable && !this.metadataFeed.length) { + } else if (this.metadata.writable && !this.metadata.length) { initialize(this._contentKeyPair) } else { restore(this._contentKeyPair) @@ -141,8 +141,8 @@ class Hyperdrive extends EventEmitter { self._ensureContent(keyPair.publicKey, err => { if (err) return cb(err) self._db = hypertrie(null, { - feed: self.metadataFeed, - metadata: self.contentFeed.key, + feed: self.metadata, + metadata: self.content.key, }) self._db.ready(function (err) { @@ -159,9 +159,9 @@ class Hyperdrive extends EventEmitter { */ function restore (keyPair) { self._db = hypertrie(null, { - feed: self.metadataFeed + feed: self.metadata }) - if (self.metadataFeed.writable) { + if (self.metadata.writable) { self._db.ready(err => { if (err) return done(err) self._ensureContent(null, done) @@ -173,8 +173,8 @@ class Hyperdrive extends EventEmitter { function done (err) { if (err) return cb(err) - self.key = self.metadataFeed.key - self.discoveryKey = self.metadataFeed.discoveryKey + self.key = self.metadata.key + self.discoveryKey = self.metadata.discoveryKey return cb(null) } @@ -201,14 +201,14 @@ class Hyperdrive extends EventEmitter { } function onkey (publicKey) { - self.contentFeed = self._createHypercore(self._storages.content, publicKey, self._contentOpts) - self.contentFeed.ready(err => { + self.content = self._createHypercore(self._storages.content, publicKey, self._contentOpts) + self.content.ready(err => { if (err) return cb(err) - self._contentFeedByteLength = self.contentFeed.byteLength - self._contentFeedLength = self.contentFeed.length + self._contentFeedByteLength = self.content.byteLength + self._contentFeedLength = self.content.length - self.contentFeed.on('error', err => self.emit('error', err)) + self.content.on('error', err => self.emit('error', err)) return cb(null) }) } @@ -217,7 +217,7 @@ class Hyperdrive extends EventEmitter { _contentReady (cb) { this.ready(err => { if (err) return cb(err) - if (this.contentFeed) return cb(null) + if (this.content) return cb(null) this._ensureContent(null, cb) }) } @@ -245,7 +245,7 @@ class Hyperdrive extends EventEmitter { open (name, flags, cb) { name = unixify(name) - FD.create(this, name, flags, (err, fd) => { + createFileDescriptor(this, name, flags, (err, fd) => { if (err) return cb(err) cb(null, STDIO_CAP + this._fds.push(fd) - 1) }) @@ -303,7 +303,7 @@ class Hyperdrive extends EventEmitter { const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) stream.start({ - feed: this.contentFeed, + feed: this.content, blockOffset: st.offset, blockLength: st.blocks, byteOffset, @@ -399,13 +399,13 @@ class Hyperdrive extends EventEmitter { if (err) return proxy.destroy(err) if (proxy.destroyed) return release() - const byteOffset = self.contentFeed.byteLength - const offset = self.contentFeed.length + const byteOffset = self.content.byteLength + const offset = self.content.length self.emit('appending', name, opts) // TODO: revert the content feed if this fails!!!! (add an option to the write stream for this (atomic: true)) - const stream = self.contentFeed.createWriteStream() + const stream = self.content.createWriteStream() proxy.on('close', ondone) proxy.on('finish', ondone) @@ -416,8 +416,8 @@ class Hyperdrive extends EventEmitter { ...opts, offset, byteOffset, - size: self.contentFeed.byteLength - byteOffset, - blocks: self.contentFeed.length - offset + size: self.content.byteLength - byteOffset, + blocks: self.content.length - offset }) try { @@ -438,8 +438,8 @@ class Hyperdrive extends EventEmitter { function ondone () { proxy.removeListener('close', ondone) proxy.removeListener('finish', ondone) - self._contentFeedLength = self.contentFeed.length - self._contentFeedByteLength = self.contentFeed.byteLength + self._contentFeedLength = self.content.length + self._contentFeedByteLength = self.content.byteLength release() } } @@ -665,12 +665,12 @@ class Hyperdrive extends EventEmitter { if (!opts) opts = {} opts.expectedFeeds = 2 - const stream = this.metadataFeed.replicate(opts) + const stream = this.metadata.replicate(opts) this.contentReady(err => { if (err) return stream.destroy(err) if (stream.destroyed) return - this.contentFeed.replicate({ + this.content.replicate({ live: opts.live, download: opts.download, upload: opts.upload, @@ -684,8 +684,8 @@ class Hyperdrive extends EventEmitter { checkout (version, opts) { opts = { ...opts, - metadataFeed: this.metadataFeed, - contentFeed: this.contentFeed, + metadata: this.metadata, + content: this.content, _db: this._db.checkout(version), } return new Hyperdrive(this.storage, this.key, opts) @@ -706,9 +706,9 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return cb(err) - this.metadataFeed.close(err => { - if (!this.contentFeed) return cb(err) - this.contentFeed.close(cb) + this.metadata.close(err => { + if (!this.content) return cb(err) + this.content.close(cb) }) }) } diff --git a/lib/content.js b/lib/content.js index 8482f635..0ec7c373 100644 --- a/lib/content.js +++ b/lib/content.js @@ -21,7 +21,7 @@ function contentOptions (self, secretKey) { maxRequests: self.maxRequests, secretKey: secretKey, storeSecretKey: false, - indexing: self.metadataFeed.writable && self.indexing, + indexing: self.metadata.writable && self.indexing, storageCacheSize: self.contentStorageCacheSize } } diff --git a/lib/fd.js b/lib/fd.js index 2af4ca39..f90833a4 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -57,27 +57,23 @@ class FileDescriptor { write (buffer, offset, len, pos, cb) { if (!this.writable) return cb(new errors.BadFileDescriptor('File descriptor not open for writing.')) if (!this.stat && !this.creating) { - return cb(new errors.BadFileDescriptor('File descriptor not open in create mode.')) + return process.nextTick(cb, new errors.BadFileDescriptor('File descriptor not open in create mode.')) } if (this.position !== null && pos !== this.position) { - return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) + return process.nextTick(cb, new errors.BadFileDescriptor('Random-access writes are not currently supported.')) } if (this.appending && pos < this.stat.size) { - return cb(new errors.BadFileDescriptor('Position cannot be less than the file size when appending.')) + return process.nextTick(cb, new errors.BadFileDescriptor('Position cannot be less than the file size when appending.')) } if (!this._writeStream && !this.appending && pos) { - return cb(new errors.BadFileDescriptor('Random-access writes are not currently supported.')) + return process.nextTick(cb, new errors.BadFileDescriptor('Random-access writes are not currently supported.')) } const self = this + if (!this._writeStream) { - const writeStream = this.drive.createWriteStream(this.path) - this._writeStream = pumpify(this._batcher, through.obj((chunk, enc, cb) => { - cb(null, Buffer.concat(chunk)) - }), writeStream) - this._writeStream.on('error', err => { - return cb(err) - }) + this._writeStream = createWriteStream(this.drive, this.path) + this._writeStream.on('error', cb) } // TODO: This is a temporary (bad) way of supporting appends. @@ -109,7 +105,7 @@ class FileDescriptor { } function write(stream, data, cb) { - if (!stream.write(data)) { + if (stream.write(data) === false) { stream.once('drain', cb) } else { process.nextTick(cb) @@ -131,7 +127,7 @@ class FileDescriptor { } } if (this._range) { - this.drive.contentFeed.undownload(this._range) + this.drive.content.undownload(this._range) this._range = null } process.nextTick(cb, null) @@ -143,21 +139,20 @@ class FileDescriptor { * TODO: This behavior should be more customizable in the future. */ _refreshDownload (start, cb) { - let end = Math.min(this.stat.blocks + this.stat.offset, start + 16) + const end = Math.min(this.stat.blocks + this.stat.offset, start + 16) if (this._range) { - this.drive.contentFeed.undownload(this._range) + this.drive.content.undownload(this._range) } - this._range = this.drive.contentFeed.download({ start, end, linear: true }, cb || noop) + this._range = this.drive.content.download({ start, end, linear: true }, cb || noop) } _seekAndRead (buffer, offset, len, pos, cb) { const start = this.stat.offset const end = start + this.stat.blocks - - this.drive.contentFeed.seek(this.stat.byteOffset + pos, { start, end }, (err, blk, blockOffset) => { + this.drive.content.seek(this.stat.byteOffset + pos, { start, end }, (err, blk, blockOffset) => { if (err) return cb(err) this.position = pos this.blockPosition = blk @@ -204,7 +199,7 @@ class FileDescriptor { return process.nextTick(cb, null, 0, buffer) } - this.drive.contentFeed.get(blk, (err, data) => { + this.drive.content.get(blk, (err, data) => { if (err) return cb(err) if (blkOffset) data = data.slice(blkOffset) @@ -226,11 +221,11 @@ class FileDescriptor { } } -FileDescriptor.create = function (drive, name, flags, cb) { +module.exports = function create (drive, name, flags, cb) { try { flags = toFlagsNumber(flags) } catch (err) { - return cb(err) + return process.nextTick(cb, err) } const accmode = flags & O_ACCMODE @@ -268,8 +263,6 @@ FileDescriptor.create = function (drive, name, flags, cb) { }) } -module.exports = FileDescriptor - // Copied from the Node FS internal utils. function toFlagsNumber (flags) { if (typeof flags === 'number') { @@ -307,3 +300,11 @@ function toFlagsNumber (flags) { throw new errors.InvalidArgument(`Invalid value in flags: ${flags}`) } + +function createWriteStream (drive, path) { + const writeStream = drive.createWriteStream(path) + const batcher = byteStream({ time: 100, limit: 4096 * 16 }) + return pumpify(batcher, through.obj((chunk, enc, cb) => { + cb(null, Buffer.concat(chunk)) + }), writeStream) +} diff --git a/lib/storage.js b/lib/storage.js index bb7c630a..2c50bacd 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -13,7 +13,7 @@ function wrap (self, storage) { } } -function defaultStorage (self, storage, opts) { +module.exports = function defaultStorage (self, storage, opts) { var folder = '' if (typeof storage === 'object' && storage) return wrap(self, storage) @@ -32,7 +32,3 @@ function defaultStorage (self, storage, opts) { } } } - -module.exports = { - defaultStorage -} diff --git a/test/basic.js b/test/basic.js index 2b977d88..307fb303 100644 --- a/test/basic.js +++ b/test/basic.js @@ -109,8 +109,8 @@ tape('provide keypair', function (t) { archive.on('ready', function () { t.ok(archive.writable) - t.ok(archive.metadataFeed.writable) - t.ok(archive.contentFeed.writable) + t.ok(archive.metadata.writable) + t.ok(archive.content.writable) t.ok(publicKey.equals(archive.key)) archive.writeFile('/hello.txt', 'world', function (err) { @@ -148,8 +148,8 @@ tape.skip('download a version', function (t) { var src = create() src.on('ready', function () { t.ok(src.writable) - t.ok(src.metadataFeed.writable) - t.ok(src.contentFeed.writable) + t.ok(src.metadata.writable) + t.ok(src.content.writable) src.writeFile('/first.txt', 'number 1', function (err) { t.error(err, 'no error') src.writeFile('/second.txt', 'number 2', function (err) { diff --git a/test/creation.js b/test/creation.js index a9fb828a..fab1af5c 100644 --- a/test/creation.js +++ b/test/creation.js @@ -6,8 +6,8 @@ tape('owner is writable', function (t) { archive.on('ready', function () { t.ok(archive.writable) - t.ok(archive.metadataFeed.writable) - t.ok(archive.contentFeed.writable) + t.ok(archive.metadata.writable) + t.ok(archive.content.writable) t.end() }) }) diff --git a/test/storage.js b/test/storage.js index 1b3f81c6..fd66c51d 100644 --- a/test/storage.js +++ b/test/storage.js @@ -7,8 +7,8 @@ tape('ram storage', function (t) { var archive = create() archive.ready(function () { - t.ok(archive.metadataFeed.writable, 'archive metadata is writable') - t.ok(archive.contentFeed.writable, 'archive content is writable') + t.ok(archive.metadata.writable, 'archive metadata is writable') + t.ok(archive.content.writable, 'archive content is writable') t.end() }) }) @@ -18,16 +18,16 @@ tape('dir storage with resume', function (t) { t.ifError(err) var archive = hyperdrive(dir) archive.ready(function () { - t.ok(archive.metadataFeed.writable, 'archive metadata is writable') - t.ok(archive.contentFeed.writable, 'archive content is writable') + t.ok(archive.metadata.writable, 'archive metadata is writable') + t.ok(archive.content.writable, 'archive content is writable') t.same(archive.version, 1, 'archive has version 1') archive.close(function (err) { t.ifError(err) var archive2 = hyperdrive(dir) archive2.ready(function (err) { - t.ok(archive2.metadataFeed.writable, 'archive2 metadata is writable') - t.ok(archive2.contentFeed.writable, 'archive2 content is writable') + t.ok(archive2.metadata.writable, 'archive2 metadata is writable') + t.ok(archive2.content.writable, 'archive2 content is writable') t.same(archive2.version, 1, 'archive has version 1') cleanup(function (err) { @@ -48,8 +48,8 @@ tape('dir storage for non-writable archive', function (t) { var clone = hyperdrive(dir, src.key) clone.on('content', function () { - t.ok(!clone.metadataFeed.writable, 'clone metadata not writable') - t.ok(!clone.contentFeed.writable, 'clone content not writable') + t.ok(!clone.metadata.writable, 'clone metadata not writable') + t.ok(!clone.content.writable, 'clone content not writable') t.same(clone.key, src.key, 'keys match') cleanup(function (err) { t.ifError(err) @@ -107,7 +107,7 @@ tape('sparse read/write two files', function (t) { t.error(err, 'no error') var stream = clone.replicate() stream.pipe(archive.replicate()).pipe(stream) - clone.metadataFeed.update(start) + clone.metadata.update(start) }) }) From 9bee54323902456c8f7d83902e59fbdbc58dfc9f Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 30 Mar 2019 21:00:23 -0700 Subject: [PATCH 049/108] Standardized --- index.js | 22 ++++++++-------------- lib/content.js | 1 - lib/fd.js | 9 ++++----- lib/stat.js | 12 ++++++------ test/basic.js | 5 ++--- test/creation.js | 2 -- test/fd.js | 27 ++++++++++++--------------- test/fuzzing.js | 16 +++++----------- test/storage.js | 1 + 9 files changed, 38 insertions(+), 57 deletions(-) diff --git a/index.js b/index.js index 413a9f5f..cc55010d 100644 --- a/index.js +++ b/index.js @@ -142,7 +142,7 @@ class Hyperdrive extends EventEmitter { if (err) return cb(err) self._db = hypertrie(null, { feed: self.metadata, - metadata: self.content.key, + metadata: self.content.key }) self._db.ready(function (err) { @@ -283,7 +283,7 @@ class Hyperdrive extends EventEmitter { const length = typeof opts.end === 'number' ? 1 + opts.end - (opts.start || 0) : typeof opts.length === 'number' ? opts.length : -1 const stream = coreByteStream({ ...opts, - highWaterMark: opts.highWaterMark || 64 * 1024 + highWaterMark: opts.highWaterMark || 64 * 1024 }) this.contentReady(err => { @@ -327,10 +327,10 @@ class Hyperdrive extends EventEmitter { if (err) return proxy.destroy(err) const decoder = through.obj((chunk, enc, cb) => { - let obj = { type: !chunk.left ? 'del' : 'put', name: chunk.key} + let obj = { type: !chunk.left ? 'del' : 'put', name: chunk.key } if (chunk.left) { try { - obj.stat = messages.Stat.decode(chunk.left.value) + obj.stat = messages.Stat.decode(chunk.left.value) } catch (err) { return cb(err) } @@ -627,7 +627,7 @@ class Hyperdrive extends EventEmitter { this._db.list(name, { gt: true, recursive }, (err, list) => { if (err) return cb(err) return cb(null, list.map(st => { - if (name === '/') return st.key.split('/')[0] + if (name === '/') return st.key.split('/')[0] return path.relative(name, st.key).split('/')[0] })) }) @@ -653,6 +653,8 @@ class Hyperdrive extends EventEmitter { if (!cb) cb = noop name = unixify(name) + const self = this + let stream = this._db.iterator(name) stream.next((err, val) => { if (err) return cb(err) @@ -686,7 +688,7 @@ class Hyperdrive extends EventEmitter { ...opts, metadata: this.metadata, content: this.content, - _db: this._db.checkout(version), + _db: this._db.checkout(version) } return new Hyperdrive(this.storage, this.key, opts) } @@ -723,12 +725,4 @@ function isObject (val) { return !!val && typeof val !== 'string' && !Buffer.isBuffer(val) } -function split (buf) { - var list = [] - for (var i = 0; i < buf.length; i += 65536) { - list.push(buf.slice(i, i + 65536)) - } - return list -} - function noop () {} diff --git a/lib/content.js b/lib/content.js index 0ec7c373..517edb10 100644 --- a/lib/content.js +++ b/lib/content.js @@ -30,4 +30,3 @@ module.exports = { contentKeyPair, contentOptions } - diff --git a/lib/fd.js b/lib/fd.js index f90833a4..895d8be5 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -1,5 +1,4 @@ const fs = require('fs') -const pump = require('pump') const errors = require('./errors') const byteStream = require('byte-stream') const through = require('through2') @@ -104,7 +103,7 @@ class FileDescriptor { }) } - function write(stream, data, cb) { + function write (stream, data, cb) { if (stream.write(data) === false) { stream.once('drain', cb) } else { @@ -120,7 +119,7 @@ class FileDescriptor { this._writeStream = null } else { return this._writeStream.end(err => { - if (err) return cb(err) + if (err) return cb(err) this._writeStream = null return cb(null) }) @@ -243,7 +242,7 @@ module.exports = function create (drive, name, flags, cb) { if (!st && (!writable || !creating)) return cb(new errors.FileNotFound(name)) if (st) { - try { + try { st = messages.Stat.decode(st.value) } catch (err) { return cb(err) @@ -305,6 +304,6 @@ function createWriteStream (drive, path) { const writeStream = drive.createWriteStream(path) const batcher = byteStream({ time: 100, limit: 4096 * 16 }) return pumpify(batcher, through.obj((chunk, enc, cb) => { - cb(null, Buffer.concat(chunk)) + cb(null, Buffer.concat(chunk)) }), writeStream) } diff --git a/lib/stat.js b/lib/stat.js index 873c59ee..79ee1588 100644 --- a/lib/stat.js +++ b/lib/stat.js @@ -62,12 +62,12 @@ Stat.directory = function (data) { } Stat.IFSOCK = 0b1100 << 12 -Stat.IFLNK = 0b1010 << 12 -Stat.IFREG = 0b1000 << 12 -Stat.IFBLK = 0b0110 << 12 -Stat.IFDIR = 0b0100 << 12 -Stat.IFCHR = 0b0010 << 12 -Stat.IFIFO = 0b0001 << 12 +Stat.IFLNK = 0b1010 << 12 +Stat.IFREG = 0b1000 << 12 +Stat.IFBLK = 0b0110 << 12 +Stat.IFDIR = 0b0100 << 12 +Stat.IFCHR = 0b0010 << 12 +Stat.IFIFO = 0b0001 << 12 function getTime (date) { if (typeof date === 'number') return date diff --git a/test/basic.js b/test/basic.js index 307fb303..2ee73320 100644 --- a/test/basic.js +++ b/test/basic.js @@ -139,7 +139,6 @@ tape('write and read, no cache', function (t) { t.end() }) }) - var self = this }) // TODO: Re-enable the following tests once the `download` and `fetchLatest` APIs are reimplemented. @@ -324,7 +323,7 @@ tape('can read nested directories', async function (t) { let rootSet = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g']) let bSet = new Set(['a', 'c']) let dSet = new Set(['e']) - let eSet = new Set(['a', 'b']) + let eSet = new Set(['a', 'b']) let deSet = new Set(['f', 'a']) for (let file of files) { @@ -364,7 +363,7 @@ tape('can read nested directories', async function (t) { }) tape('can read sparse metadata', async function (t) { - const { read, write } = await getTestDrives() + const { read, write } = await getTestDrives() let files = ['a', 'b/a/b', 'b/c', 'c/b', 'd/e/f/g/h', 'd/e/a', 'e/a', 'e/b', 'f', 'g'] diff --git a/test/creation.js b/test/creation.js index fab1af5c..15c861f2 100644 --- a/test/creation.js +++ b/test/creation.js @@ -11,5 +11,3 @@ tape('owner is writable', function (t) { t.end() }) }) - - diff --git a/test/fd.js b/test/fd.js index b7fefd6c..4f87a3d8 100644 --- a/test/fd.js +++ b/test/fd.js @@ -1,10 +1,9 @@ const tape = require('tape') const create = require('./helpers/create') -const hyperdrive = require('..') tape('basic fd read', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') drive.writeFile('hi', content, function (err) { t.error(err, 'no error') @@ -48,7 +47,7 @@ tape('basic fd read', function (t) { tape('basic fd read with implicit position', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') drive.writeFile('hi', content, function (err) { t.error(err, 'no error') @@ -92,7 +91,7 @@ tape('basic fd read with implicit position', function (t) { tape('fd read with zero length', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') drive.writeFile('hi', content, function (err) { t.error(err, 'no error') @@ -113,7 +112,7 @@ tape('fd read with zero length', function (t) { tape('fd read with out-of-bounds offset', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') drive.writeFile('hi', content, function (err) { t.error(err, 'no error') @@ -134,7 +133,7 @@ tape('fd read with out-of-bounds offset', function (t) { tape('fd read with out-of-bounds length', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') drive.writeFile('hi', content, function (err) { t.error(err, 'no error') @@ -155,8 +154,6 @@ tape('fd read with out-of-bounds length', function (t) { tape('fd read of empty drive', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') - drive.open('hi', 'r', function (err, fd) { t.true(err) t.same(err.errno, 2) @@ -166,7 +163,7 @@ tape('fd read of empty drive', function (t) { tape('fd read of invalid file', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') drive.writeFile('hi', content, function (err) { t.error(err, 'no error') @@ -180,7 +177,7 @@ tape('fd read of invalid file', function (t) { tape('fd basic write, creating file', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') drive.open('hello', 'w+', function (err, fd) { t.error(err, 'no error') drive.write(fd, content, 0, content.length, 0, function (err, bytesWritten) { @@ -200,7 +197,7 @@ tape('fd basic write, creating file', function (t) { tape('fd basic write, appending file', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) let second = content.slice(2000) @@ -230,7 +227,7 @@ tape('fd basic write, appending file', function (t) { tape('fd basic write, overwrite file', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) let second = content.slice(2000) @@ -260,7 +257,7 @@ tape('fd basic write, overwrite file', function (t) { tape('fd stateful write', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) let second = content.slice(2000) @@ -291,7 +288,7 @@ tape('huge stateful write + stateless read', function (t) { const drive = create() const content = Buffer.alloc(SLICE_SIZE * NUM_SLICES).fill('0123456789abcdefghijklmnopqrstuvwxyz') - let slices = new Array(NUM_SLICES).fill(0).map((_, i) => content.slice(SLICE_SIZE * i, SLICE_SIZE * (i+1))) + let slices = new Array(NUM_SLICES).fill(0).map((_, i) => content.slice(SLICE_SIZE * i, SLICE_SIZE * (i + 1))) drive.open('hello', 'w+', function (err, fd) { t.error(err, 'no error') @@ -341,7 +338,7 @@ tape('huge stateful write + stateless read', function (t) { tape('fd random-access write fails', function (t) { const drive = create() - const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') + const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) let second = content.slice(2000) diff --git a/test/fuzzing.js b/test/fuzzing.js index 2a1fc4c5..c4578c16 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -1,8 +1,7 @@ const tape = require('tape') -const sodium = require('sodium-universal') const collect = require('stream-collector') -const FuzzBuzz = require('fuzzbuzz') +const FuzzBuzz = require('fuzzbuzz') const create = require('./helpers/create') const MAX_PATH_DEPTH = 30 @@ -33,7 +32,7 @@ class HyperdriveFuzzer extends FuzzBuzz { // START Helper functions. _select (map) { - let idx = this.randomInt(map.size -1) + let idx = this.randomInt(map.size - 1) if (idx < 0) return null let ite = map.entries() @@ -84,7 +83,7 @@ class HyperdriveFuzzer extends FuzzBuzz { // START FuzzBuzz interface _setup () { - this.drive = create() + this.drive = create() this.files = new Map() this.directories = new Map() this.streams = new Map() @@ -138,7 +137,6 @@ class HyperdriveFuzzer extends FuzzBuzz { } } - async call (ops) { let res = await super.call(ops) this.log.push(res) @@ -218,7 +216,7 @@ class HyperdriveFuzzer extends FuzzBuzz { existingFileOverwrite () { let selected = this._selectFile() if (!selected) return - let [fileName, content] = selected + let [fileName] = selected let { content: newContent } = this._createFile() @@ -390,7 +388,7 @@ class HyperdriveFuzzer extends FuzzBuzz { return writeNext(fd) }) - function writeNext(fd) { + function writeNext (fd) { let next = bufs[count] self.debug(` Writing content with length ${next.length} to FD ${fd} at pos: ${pos}`) self.drive.write(fd, next, 0, next.length, pos, (err, bytesWritten) => { @@ -446,13 +444,9 @@ class HyperdriveFuzzer extends FuzzBuzz { } }) } - } class SparseHyperdriveFuzzer extends HyperdriveFuzzer { - constructor(opts) { - super(opts) - } async _setup () { await super._setup() diff --git a/test/storage.js b/test/storage.js index fd66c51d..11542ce1 100644 --- a/test/storage.js +++ b/test/storage.js @@ -26,6 +26,7 @@ tape('dir storage with resume', function (t) { var archive2 = hyperdrive(dir) archive2.ready(function (err) { + t.error(err, 'no error') t.ok(archive2.metadata.writable, 'archive2 metadata is writable') t.ok(archive2.content.writable, 'archive2 content is writable') t.same(archive2.version, 1, 'archive has version 1') From e1268afc5572656b8705384dd1f8d9e5a83b757c Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 9 Apr 2019 05:05:07 -0700 Subject: [PATCH 050/108] Use filesystem-constants --- lib/fd.js | 57 +++++++++------------------------------------------- package.json | 1 + 2 files changed, 11 insertions(+), 47 deletions(-) diff --git a/lib/fd.js b/lib/fd.js index 895d8be5..6d130f00 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -1,12 +1,11 @@ -const fs = require('fs') -const errors = require('./errors') const byteStream = require('byte-stream') const through = require('through2') const pumpify = require('pumpify') -const messages = require('./messages') -const noop = () => {} +const messages = require('./messages') +const errors = require('./errors') +const { linux: linuxConstants, parse } = require('filesystem-constants') const { O_RDONLY, O_WRONLY, @@ -15,9 +14,9 @@ const { O_TRUNC, O_APPEND, O_SYNC, - O_EXCL -} = fs.constants -const O_ACCMODE = 3 + O_EXCL, + O_ACCMODE, +} = linuxConstants class FileDescriptor { constructor (drive, path, stat, readable, writable, appending, creating) { @@ -222,9 +221,9 @@ class FileDescriptor { module.exports = function create (drive, name, flags, cb) { try { - flags = toFlagsNumber(flags) + flags = parse(linuxConstants, flags) } catch (err) { - return process.nextTick(cb, err) + return process.nextTick(cb, new errors.InvalidArgument(err.message)) } const accmode = flags & O_ACCMODE @@ -262,44 +261,6 @@ module.exports = function create (drive, name, flags, cb) { }) } -// Copied from the Node FS internal utils. -function toFlagsNumber (flags) { - if (typeof flags === 'number') { - return flags - } - - switch (flags) { - case 'r' : return O_RDONLY - case 'rs' : // Fall through. - case 'sr' : return O_RDONLY | O_SYNC - case 'r+' : return O_RDWR - case 'rs+' : // Fall through. - case 'sr+' : return O_RDWR | O_SYNC - - case 'w' : return O_TRUNC | O_CREAT | O_WRONLY - case 'wx' : // Fall through. - case 'xw' : return O_TRUNC | O_CREAT | O_WRONLY | O_EXCL - - case 'w+' : return O_TRUNC | O_CREAT | O_RDWR - case 'wx+': // Fall through. - case 'xw+': return O_TRUNC | O_CREAT | O_RDWR | O_EXCL - - case 'a' : return O_APPEND | O_CREAT | O_WRONLY - case 'ax' : // Fall through. - case 'xa' : return O_APPEND | O_CREAT | O_WRONLY | O_EXCL - case 'as' : // Fall through. - case 'sa' : return O_APPEND | O_CREAT | O_WRONLY | O_SYNC - - case 'a+' : return O_APPEND | O_CREAT | O_RDWR - case 'ax+': // Fall through. - case 'xa+': return O_APPEND | O_CREAT | O_RDWR | O_EXCL - case 'as+': // Fall through. - case 'sa+': return O_APPEND | O_CREAT | O_RDWR | O_SYNC - } - - throw new errors.InvalidArgument(`Invalid value in flags: ${flags}`) -} - function createWriteStream (drive, path) { const writeStream = drive.createWriteStream(path) const batcher = byteStream({ time: 100, limit: 4096 * 16 }) @@ -307,3 +268,5 @@ function createWriteStream (drive, path) { cb(null, Buffer.concat(chunk)) }), writeStream) } + +function noop () {} diff --git a/package.json b/package.json index 803332c9..4e3f9361 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "byte-stream": "^2.1.0", "custom-error-class": "^1.0.0", "duplexify": "^3.7.1", + "filesystem-constants": "^1.0.0", "hypercore": "^6.25.0", "hypercore-byte-stream": "^1.0.2", "hypertrie": "^3.4.0", From dd62a854beace602b09f9c6d5eb97909e0354f01 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 9 Apr 2019 05:06:34 -0700 Subject: [PATCH 051/108] var -> let --- lib/storage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/storage.js b/lib/storage.js index 2c50bacd..dd29a27b 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -14,7 +14,7 @@ function wrap (self, storage) { } module.exports = function defaultStorage (self, storage, opts) { - var folder = '' + let folder = '' if (typeof storage === 'object' && storage) return wrap(self, storage) From 3877eec49e6c0244b0df62d57cbc9e57b4d9224e Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 10 Apr 2019 01:53:24 -0700 Subject: [PATCH 052/108] 10.0.0-rc0 --- index.js | 1 + lib/content.js | 7 +++---- package.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index cc55010d..7c059658 100644 --- a/index.js +++ b/index.js @@ -25,6 +25,7 @@ const { contentKeyPair, contentOptions } = require('./lib/content') const STDIO_CAP = 20 module.exports = (...args) => new Hyperdrive(...args) +module.exports.constants = require('filesystem-constants').linux class Hyperdrive extends EventEmitter { constructor (storage, key, opts) { diff --git a/lib/content.js b/lib/content.js index 517edb10..06767c65 100644 --- a/lib/content.js +++ b/lib/content.js @@ -1,16 +1,15 @@ const sodium = require('sodium-universal') function contentKeyPair (secretKey) { - var seed = Buffer.allocUnsafe(sodium.crypto_sign_SEEDBYTES) - var context = Buffer.from('hyperdri', 'utf8') // 8 byte context - var keyPair = { + let seed = Buffer.allocUnsafe(sodium.crypto_sign_SEEDBYTES) + let context = Buffer.from('hyperdri', 'utf8') // 8 byte context + let keyPair = { publicKey: Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES), secretKey: Buffer.allocUnsafe(sodium.crypto_sign_SECRETKEYBYTES) } sodium.crypto_kdf_derive_from_key(seed, 1, context, secretKey) sodium.crypto_sign_seed_keypair(keyPair.publicKey, keyPair.secretKey, seed) - if (seed.fill) seed.fill(0) return keyPair } diff --git a/package.json b/package.json index 4e3f9361..290e756a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "9.14.3", + "version": "10.0.0-rc0", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From 8d12c17a117b3f541102968260783afeb6a70ecf Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 17 Apr 2019 16:08:48 +0200 Subject: [PATCH 053/108] Update FD size during write (#237) * Update stat.size during fd write * Create-mode FD sets the stat after creation * Remove console.log --- index.js | 6 +++++- lib/fd.js | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 7c059658..1b4dca25 100644 --- a/index.js +++ b/index.js @@ -64,6 +64,7 @@ class Hyperdrive extends EventEmitter { this._contentFeedByteLength = null this._lock = mutexify() this._fds = [] + this._writingFd = null this.ready = thunky(this._ready.bind(this)) this.contentReady = thunky(this._contentReady.bind(this)) @@ -459,7 +460,7 @@ class Hyperdrive extends EventEmitter { } this._db.put(name, st, err => { if (err) return cb(err) - return cb(null) + return cb(null, st) }) }) } @@ -586,6 +587,9 @@ class Hyperdrive extends EventEmitter { } catch (err) { return cb(err) } + if (this._writingFd && name === this._writingFd.path) { + st.size = this._writingFd.stat.size + } cb(null, new Stat(st)) }) }) diff --git a/lib/fd.js b/lib/fd.js index 6d130f00..546d4f9d 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -71,6 +71,7 @@ class FileDescriptor { if (!this._writeStream) { this._writeStream = createWriteStream(this.drive, this.path) + this.drive._writingFd = this this._writeStream.on('error', cb) } @@ -98,6 +99,7 @@ class FileDescriptor { write(self._writeStream, slice, err => { if (err) return cb(err) self.position += slice.length + self.stat.size += slice.length return cb(null, slice.length, buffer) }) } @@ -113,6 +115,7 @@ class FileDescriptor { close (cb) { // TODO: undownload initial range + this.drive._writingFd = null if (this._writeStream) { if (this._writeStream.destroyed) { this._writeStream = null @@ -250,8 +253,9 @@ module.exports = function create (drive, name, flags, cb) { const fd = new FileDescriptor(drive, name, st, readable, writable, appending, creating) if (creating) { - drive.create(name, err => { + drive.create(name, (err, st) => { if (err) return cb(err) + fd.stat = st return cb(null, fd) }) } else { From 0bb221041f545a6e92ae022ac467f297e46165f2 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 17 Apr 2019 16:35:15 +0200 Subject: [PATCH 054/108] 10.0.0-rc1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 290e756a..4737e70b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc0", + "version": "10.0.0-rc1", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From 4700e26288d9a294789ebd71639911da101ddd48 Mon Sep 17 00:00:00 2001 From: Mathias Buus Date: Fri, 19 Apr 2019 23:02:35 +0200 Subject: [PATCH 055/108] fix create returning a buffer instead of a stat obj --- index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 1b4dca25..a603422c 100644 --- a/index.js +++ b/index.js @@ -454,11 +454,12 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return cb(err) try { - var st = messages.Stat.encode(Stat.file(opts)) + var st = Stat.file(opts) + var buf = messages.Stat.encode(st) } catch (err) { return cb(err) } - this._db.put(name, st, err => { + this._db.put(name, buf, err => { if (err) return cb(err) return cb(null, st) }) From 11c179bb4e713f872ff7b149ebf3e1277e5deda7 Mon Sep 17 00:00:00 2001 From: Mathias Buus Date: Fri, 19 Apr 2019 23:04:48 +0200 Subject: [PATCH 056/108] 10.0.0-rc2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4737e70b..4bc371b9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc1", + "version": "10.0.0-rc2", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From 854ea780c3be1054da5bb90dbd4597223dc98626 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 23 Apr 2019 13:10:27 +0200 Subject: [PATCH 057/108] Pass stat through FD createWriteStream + fix create when file exists --- index.js | 43 +++++++++++++++++++++---------------------- lib/fd.js | 6 +++--- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/index.js b/index.js index a603422c..b4cd67eb 100644 --- a/index.js +++ b/index.js @@ -453,15 +453,19 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return cb(err) - try { - var st = Stat.file(opts) - var buf = messages.Stat.encode(st) - } catch (err) { - return cb(err) - } - this._db.put(name, buf, err => { - if (err) return cb(err) - return cb(null, st) + this.lstat(name, (err, stat) => { + if (err && err.errno !== 2) return cb(err) + if (stat) return cb(null, stat) + try { + var st = Stat.file(opts) + var buf = messages.Stat.encode(st) + } catch (err) { + return cb(err) + } + this._db.put(name, buf, err => { + if (err) return cb(err) + return cb(null, st) + }) }) }) } @@ -500,14 +504,9 @@ class Hyperdrive extends EventEmitter { this.contentReady(err => { if (err) return cb(err) - this._db.get(name, (err, st) => { - if (err) return cb(err) - if (!st || !size) return this.create(name, cb) - try { - st = messages.Stat.decode(st.value) - } catch (err) { - return cb(err) - } + this.lstat(name, (err, st) => { + if (err && err.errno !== 2) return cb(err) + if (!st) return this.create(name, cb) if (size === st.size) return cb(null) if (size < st.size) { const readStream = this.createReadStream(name, { length: size }) @@ -550,11 +549,6 @@ class Hyperdrive extends EventEmitter { condition: ifNotExists }, cb) }) - - function ifNotExists (oldNode, newNode, cb) { - if (oldNode) return cb(new errors.PathAlreadyExists(name)) - return cb(null, true) - } } _statDirectory (name, opts, cb) { @@ -731,4 +725,9 @@ function isObject (val) { return !!val && typeof val !== 'string' && !Buffer.isBuffer(val) } +function ifNotExists (oldNode, newNode, cb) { + if (oldNode) return cb(new errors.PathAlreadyExists(oldNode.key)) + return cb(null, true) +} + function noop () {} diff --git a/lib/fd.js b/lib/fd.js index 546d4f9d..c1e9df93 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -70,7 +70,7 @@ class FileDescriptor { const self = this if (!this._writeStream) { - this._writeStream = createWriteStream(this.drive, this.path) + this._writeStream = createWriteStream(this.drive, this.stat, this.path) this.drive._writingFd = this this._writeStream.on('error', cb) } @@ -265,8 +265,8 @@ module.exports = function create (drive, name, flags, cb) { }) } -function createWriteStream (drive, path) { - const writeStream = drive.createWriteStream(path) +function createWriteStream (drive, opts, path) { + const writeStream = drive.createWriteStream(path, opts) const batcher = byteStream({ time: 100, limit: 4096 * 16 }) return pumpify(batcher, through.obj((chunk, enc, cb) => { cb(null, Buffer.concat(chunk)) From b158ba80a1389b469d6edbfe7c22324a739c3b11 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 23 Apr 2019 14:40:53 +0200 Subject: [PATCH 058/108] Add file option to lstat --- index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index b4cd67eb..ba7a710d 100644 --- a/index.js +++ b/index.js @@ -453,7 +453,7 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return cb(err) - this.lstat(name, (err, stat) => { + this.lstat(name, { file: true }, (err, stat) => { if (err && err.errno !== 2) return cb(err) if (stat) return cb(null, stat) try { @@ -504,7 +504,7 @@ class Hyperdrive extends EventEmitter { this.contentReady(err => { if (err) return cb(err) - this.lstat(name, (err, st) => { + this.lstat(name, { file: true }, (err, st) => { if (err && err.errno !== 2) return cb(err) if (!st) return this.create(name, cb) if (size === st.size) return cb(null) @@ -576,6 +576,7 @@ class Hyperdrive extends EventEmitter { this._db.get(name, opts, (err, node) => { if (err) return cb(err) + if (!node && opts.file) return cb(new errors.FileNotFound(name)) if (!node) return this._statDirectory(name, opts, cb) try { var st = messages.Stat.decode(node.value) From e33419033abfadbec55efed6a9f624a48d458718 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 24 Apr 2019 13:40:37 +0200 Subject: [PATCH 059/108] 1.0.0-rc3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4bc371b9..58832111 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc2", + "version": "10.0.0-rc3", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From 77ee7ae50a31a414c2470dc6768f6bfaf39e6e72 Mon Sep 17 00:00:00 2001 From: Mathias Buus Date: Mon, 29 Apr 2019 11:34:07 +0200 Subject: [PATCH 060/108] bump to 7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4bc371b9..ff3ed65e 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "custom-error-class": "^1.0.0", "duplexify": "^3.7.1", "filesystem-constants": "^1.0.0", - "hypercore": "^6.25.0", + "hypercore": "^7.0.0", "hypercore-byte-stream": "^1.0.2", "hypertrie": "^3.4.0", "mutexify": "^1.2.0", From 92b25ffa44b1cccf6f51c745d606b7293bf4476e Mon Sep 17 00:00:00 2001 From: Karissa McKelvey Date: Thu, 2 May 2019 02:15:37 -0700 Subject: [PATCH 061/108] fix: allow content feed to be passed in constructor (#238) --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index ba7a710d..b032e2a7 100644 --- a/index.js +++ b/index.js @@ -203,7 +203,7 @@ class Hyperdrive extends EventEmitter { } function onkey (publicKey) { - self.content = self._createHypercore(self._storages.content, publicKey, self._contentOpts) + self.content = self.content || self._createHypercore(self._storages.content, publicKey, self._contentOpts) self.content.ready(err => { if (err) return cb(err) From baca0b6efede807320e495b7b7a76f38d4df968a Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 8 May 2019 14:44:25 +0200 Subject: [PATCH 062/108] Moving to corestore --- index.js | 281 +++++++++++++++++++++++------------------------ lib/content.js | 13 ++- lib/fd.js | 4 +- lib/storage.js | 34 ------ package.json | 4 +- schema.proto | 1 + test/basic.js | 211 ++++------------------------------- test/checkout.js | 89 +++++++++++++++ test/deletion.js | 18 +++ test/diff.js | 58 ++++++++++ test/index.js | 10 ++ test/watch.js | 30 +++++ 12 files changed, 380 insertions(+), 373 deletions(-) delete mode 100644 lib/storage.js create mode 100644 test/checkout.js create mode 100644 test/deletion.js create mode 100644 test/diff.js create mode 100644 test/index.js create mode 100644 test/watch.js diff --git a/index.js b/index.js index ba7a710d..1c622ca6 100644 --- a/index.js +++ b/index.js @@ -11,15 +11,15 @@ const pumpify = require('pumpify') const pump = require('pump') const hypercore = require('hypercore') -const hypertrie = require('hypertrie') const coreByteStream = require('hypercore-byte-stream') +const MountableHypertrie = require('mountable-hypertrie') const createFileDescriptor = require('./lib/fd') const Stat = require('./lib/stat') const errors = require('./lib/errors') const messages = require('./lib/messages') -const defaultStorage = require('./lib/storage') -const { contentKeyPair, contentOptions } = require('./lib/content') +const defaultCorestore = require('random-access-corestore') +const { contentKeyPair, contentOptions, ContentState } = require('./lib/content') // 20 is arbitrary, just to make the fds > stdio etc const STDIO_CAP = 20 @@ -37,40 +37,34 @@ class Hyperdrive extends EventEmitter { } if (!opts) opts = {} + this.opts = opts this.key = null this.discoveryKey = null this.live = true - this.latest = !!opts.latest this.sparse = !!opts.sparse this.sparseMetadata = !!opts.sparseMetadata - this._factory = opts.factory ? storage : null - this._storages = !this.factory ? defaultStorage(this, storage, opts) : null - - this.metadata = opts.metadata || this._createHypercore(this._storages.metadata, key, { + this._corestore = opts.corestore || defaultCorestore(path => storage(path), opts) + this.metadata = this._corestore.get({ + key, + main: true, secretKey: opts.secretKey, sparse: this.sparseMetadata, createIfMissing: opts.createIfMissing, storageCacheSize: opts.metadataStorageCacheSize, valueEncoding: 'binary' }) - this._db = opts._db - this.content = opts.content || null - this.storage = storage - this.contentStorageCacheSize = opts.contentStorageCacheSize - - this._contentOpts = null - this._contentFeedLength = null - this._contentFeedByteLength = null - this._lock = mutexify() + console.log('METADATA:', this.metadata) + this._db = opts._db || new MountableHypertrie(this._corestore, key, { feed: this.metadata }) + + this._contentFeeds = new WeakMap() + if (opts.content) this._contentFeeds.set(this._db, new ContentState(opts.content)) + this._fds = [] - this._writingFd = null + this._writingFds = new Map() this.ready = thunky(this._ready.bind(this)) - this.contentReady = thunky(this._contentReady.bind(this)) - this.ready(onReady) - this.contentReady(onContentReady) const self = this @@ -78,16 +72,6 @@ class Hyperdrive extends EventEmitter { if (err) return self.emit('error', err) self.emit('ready') } - - function onContentReady (err) { - if (err) return self.emit('error', err) - self.emit('content') - } - } - - _createHypercore (storage, key, opts) { - if (this._factory) return this._factory(key, opts) - return hypercore(storage, key, opts) } get version () { @@ -100,23 +84,29 @@ class Hyperdrive extends EventEmitter { } _ready (cb) { + console.log('IN _READY') const self = this - this.metadata.on('error', onerror) - this.metadata.on('append', update) + self.metadata.on('error', onerror) + self.metadata.on('append', update) - this.metadata.ready(err => { + return self.metadata.ready(err => { if (err) return cb(err) - if (this.sparseMetadata) { - this.metadata.update(function loop () { + console.log('METADATA IS READY') + + if (self.sparseMetadata) { + self.metadata.update(function loop () { self.metadata.update(loop) }) } - this._contentKeyPair = this.metadata.secretKey ? contentKeyPair(this.metadata.secretKey) : {} - this._contentOpts = contentOptions(this, this._contentKeyPair.secretKey) - this._contentOpts.keyPair = this._contentKeyPair + const rootContentKeyPair = self.metadata.secretKey ? contentKeyPair(self.metadata.secretKey) : {} + const rootContentOpts = contentOptions(self, rootContentKeyPair.secretKey) + rootContentOpts.keyPair = rootContentKeyPair + console.log('rootContentOpts:', rootContentOpts) + + console.log('self.metadata:', self.metadata) /** * If a db is provided as input, ensure that a contentFeed is also provided, then return (this is a checkout). @@ -126,13 +116,14 @@ class Hyperdrive extends EventEmitter { * If the metadata feed is readable: * Initialize the db without metadata and load the content feed key from the header. */ - if (this._db) { - if (!this.content || !this.metadata) return cb(new Error('Must provide a db and both content/metadata feeds')) + if (self.opts._db) { + console.log('A _DB WAS PROVIDED IN OPTS') + if (!self.contentFeeds.get(self.opts._db.key)) return cb(new Error('Must provide a db and a content feed')) return done(null) - } else if (this.metadata.writable && !this.metadata.length) { - initialize(this._contentKeyPair) + } else if (self.metadata.writable && !self.metadata.length) { + initialize(rootContentKeyPair) } else { - restore(this._contentKeyPair) + restore(rootContentKeyPair) } }) @@ -140,14 +131,11 @@ class Hyperdrive extends EventEmitter { * The first time the hyperdrive is created, we initialize both the db (metadata feed) and the content feed here. */ function initialize (keyPair) { - self._ensureContent(keyPair.publicKey, err => { + console.log('INITIALIZING') + self._db.setMetadata(keyPair.publicKey) + self._db.ready(err => { if (err) return cb(err) - self._db = hypertrie(null, { - feed: self.metadata, - metadata: self.content.key - }) - - self._db.ready(function (err) { + self._getContent(self._db, { secretKey: keyPair.secretKey }, err => { if (err) return cb(err) return done(null) }) @@ -160,13 +148,12 @@ class Hyperdrive extends EventEmitter { * (Otherwise, we need to read the feed's metadata block first) */ function restore (keyPair) { - self._db = hypertrie(null, { - feed: self.metadata - }) + console.log('RESTORING') if (self.metadata.writable) { + console.log('IT IS WRITABLE') self._db.ready(err => { if (err) return done(err) - self._ensureContent(null, done) + self._getContent(self._db, done) }) } else { self._db.ready(done) @@ -174,6 +161,7 @@ class Hyperdrive extends EventEmitter { } function done (err) { + console.log('DONE WITH READY, err:', err) if (err) return cb(err) self.key = self.metadata.key self.discoveryKey = self.metadata.discoveryKey @@ -189,41 +177,34 @@ class Hyperdrive extends EventEmitter { } } - _ensureContent (publicKey, cb) { - let self = this + _getContent (db, opts, cb) { + if (typeof opts === 'function') return this._getContent(db, null, opts) + const self = this - if (publicKey) return onkey(publicKey) - else loadkey() + console.log('GETTING CONTENT FOR:', 'db:', db) + const existingContent = self._contentFeeds.get(db) + if (existingContent) return process.nextTick(cb, null, existingContent) + console.log(' NOT EXISTING') - function loadkey () { - self._db.getMetadata((err, contentKey) => { - if (err) return cb(err) - return onkey(contentKey) - }) - } + db.getMetadata((err, publicKey) => { + if (err) return cb(err) + console.log(' METADATA:', publicKey) + return onkey(publicKey) + }) function onkey (publicKey) { - self.content = self._createHypercore(self._storages.content, publicKey, self._contentOpts) - self.content.ready(err => { + const feed = self._corestore.get({ key: publicKey, ...self._contentOpts, ...opts }) + feed.ready(err => { if (err) return cb(err) - - self._contentFeedByteLength = self.content.byteLength - self._contentFeedLength = self.content.length - - self.content.on('error', err => self.emit('error', err)) - return cb(null) + const state = new ContentState(feed, mutexify()) + console.log(' SETTING CONTENT FEED FOR:', db) + self._contentFeeds.set(db, state) + feed.on('error', err => self.emit('error', err)) + return cb(null, state) }) } } - _contentReady (cb) { - this.ready(err => { - if (err) return cb(err) - if (this.content) return cb(null) - this._ensureContent(null, cb) - }) - } - _update (name, stat, cb) { name = unixify(name) @@ -278,6 +259,7 @@ class Hyperdrive extends EventEmitter { } createReadStream (name, opts) { + console.log('IN CREATEREADSTREAM FOR:', name) if (!opts) opts = {} name = unixify(name) @@ -288,35 +270,44 @@ class Hyperdrive extends EventEmitter { highWaterMark: opts.highWaterMark || 64 * 1024 }) - this.contentReady(err => { + this.ready(err => { if (err) return stream.destroy(err) - - this._db.get(name, (err, st) => { + this._db.get(name, (err, node, trie) => { if (err) return stream.destroy(err) - if (!st) return stream.destroy(new errors.FileNotFound(name)) + console.log('HERE ERR:', err, 'NODE:', node) + if (!node) return stream.destroy(new errors.FileNotFound(name)) - try { - st = messages.Stat.decode(st.value) - } catch (err) { - return stream.destroy(err) - } - - const byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset - const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) - - stream.start({ - feed: this.content, - blockOffset: st.offset, - blockLength: st.blocks, - byteOffset, - byteLength + this._getContent(trie, (err, contentState) => { + if (err) return stream.destroy(err) + return oncontent(node.value, contentState) }) }) }) + function oncontent (st, contentState) { + try { + st = messages.Stat.decode(st) + } catch (err) { + return stream.destroy(err) + } + + const byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset + const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) + + stream.start({ + feed: contentState.feed, + blockOffset: st.offset, + blockLength: st.blocks, + byteOffset, + byteLength + }) + } + return stream } + // TODO: Update when support is added to MountableHypertrie + /* createDiffStream (other, prefix, opts) { if (other instanceof Hyperdrive) other = other.version if (typeof prefix === 'object') return this.createDiffStream(other, '/', prefix) @@ -344,6 +335,7 @@ class Hyperdrive extends EventEmitter { return proxy } + */ createDirectoryStream (name, opts) { if (!opts) opts = {} @@ -387,27 +379,33 @@ class Hyperdrive extends EventEmitter { // TODO: support piping through a "split" stream like rabin - this.contentReady(err => { + this.ready(err => { if (err) return proxy.destroy(err) - this._lock(_release => { - release = _release - append() + this._db.get(name, (err, node, trie) => { + if (err) return proxy.destroy(err) + this._getContent(trie, (err, contentState) => { + if (err) return proxy.destroy(err) + console.log('contentState:', contentState) + contentState.lock(_release => { + release = _release + append(contentState) + }) + }) }) }) return proxy - function append (err) { - if (err) return proxy.destroy(err) + function append (contentState) { if (proxy.destroyed) return release() - const byteOffset = self.content.byteLength - const offset = self.content.length + const byteOffset = contentState.feed.byteLength + const offset = contentState.feed.length self.emit('appending', name, opts) // TODO: revert the content feed if this fails!!!! (add an option to the write stream for this (atomic: true)) - const stream = self.content.createWriteStream() + const stream = contentState.feed.createWriteStream() proxy.on('close', ondone) proxy.on('finish', ondone) @@ -418,8 +416,8 @@ class Hyperdrive extends EventEmitter { ...opts, offset, byteOffset, - size: self.content.byteLength - byteOffset, - blocks: self.content.length - offset + size: contentState.feed.byteLength - byteOffset, + blocks: contentState.feed.length - offset }) try { @@ -432,6 +430,7 @@ class Hyperdrive extends EventEmitter { self._db.put(name, encoded, function (err) { if (err) return proxy.destroy(err) self.emit('append', name, opts) + console.log('AFTER WRite FOR:', name, 'METADATA:', self.metadata) proxy.uncork() }) }) @@ -440,8 +439,6 @@ class Hyperdrive extends EventEmitter { function ondone () { proxy.removeListener('close', ondone) proxy.removeListener('finish', ondone) - self._contentFeedLength = self.content.length - self._contentFeedByteLength = self.content.byteLength release() } } @@ -534,20 +531,23 @@ class Hyperdrive extends EventEmitter { name = unixify(name) - this.contentReady(err => { + this._db.get(name, (err, node, trie) => { if (err) return cb(err) - try { - var st = messages.Stat.encode(Stat.directory({ - ...opts, - offset: this._contentFeedLength, - byteOffset: this._contentFeedByteLength - })) - } catch (err) { - return cb(err) - } - this._db.put(name, st, { - condition: ifNotExists - }, cb) + this._getContent(trie, (err, contentState) => { + if (err) return cb(err) + try { + var st = messages.Stat.encode(Stat.directory({ + ...opts, + offset: contentState.length, + byteOffset: contentState.byteLength + })) + } catch (err) { + return cb(err) + } + this._db.put(name, st, { + condition: ifNotExists + }, cb) + }) }) } @@ -583,8 +583,9 @@ class Hyperdrive extends EventEmitter { } catch (err) { return cb(err) } - if (this._writingFd && name === this._writingFd.path) { - st.size = this._writingFd.stat.size + const writingFd = this._writingFds.get(name) + if (writingFd) { + st.size = writingFd.stat.size } cb(null, new Stat(st)) }) @@ -665,30 +666,14 @@ class Hyperdrive extends EventEmitter { } replicate (opts) { - if (!opts) opts = {} - opts.expectedFeeds = 2 - - const stream = this.metadata.replicate(opts) - - this.contentReady(err => { - if (err) return stream.destroy(err) - if (stream.destroyed) return - this.content.replicate({ - live: opts.live, - download: opts.download, - upload: opts.upload, - stream: stream - }) - }) - + const stream = this._corestore.replicate(opts) + stream.on('error', err => console.error('REPLICATION ERROR:', err)) return stream } checkout (version, opts) { opts = { ...opts, - metadata: this.metadata, - content: this.content, _db: this._db.checkout(version) } return new Hyperdrive(this.storage, this.key, opts) @@ -720,6 +705,10 @@ class Hyperdrive extends EventEmitter { name = unixify(name) return this._db.watch(name, onchange) } + + mount (path, key, opts, cb) { + return this._db.mount(path, key, opts, cb) + } } function isObject (val) { diff --git a/lib/content.js b/lib/content.js index 06767c65..7f091c20 100644 --- a/lib/content.js +++ b/lib/content.js @@ -25,7 +25,18 @@ function contentOptions (self, secretKey) { } } +class ContentState { + constructor (feed, lock) { + this.feed = feed + this._lock = lock + } + lock (cb) { + return this._lock(cb) + } +} + module.exports = { contentKeyPair, - contentOptions + contentOptions, + ContentState } diff --git a/lib/fd.js b/lib/fd.js index c1e9df93..24705116 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -71,7 +71,7 @@ class FileDescriptor { if (!this._writeStream) { this._writeStream = createWriteStream(this.drive, this.stat, this.path) - this.drive._writingFd = this + this.drive._writingFds.set(this.path, this) this._writeStream.on('error', cb) } @@ -115,7 +115,7 @@ class FileDescriptor { close (cb) { // TODO: undownload initial range - this.drive._writingFd = null + this.drive._writingFds.delete(this.path) if (this._writeStream) { if (this._writeStream.destroyed) { this._writeStream = null diff --git a/lib/storage.js b/lib/storage.js deleted file mode 100644 index dd29a27b..00000000 --- a/lib/storage.js +++ /dev/null @@ -1,34 +0,0 @@ -const path = require('path') - -const raf = require('random-access-file') - -function wrap (self, storage) { - return { - metadata: function (name, opts) { - return storage.metadata(name, opts, self) - }, - content: function (name, opts) { - return storage.content(name, opts, self) - } - } -} - -module.exports = function defaultStorage (self, storage, opts) { - let folder = '' - - if (typeof storage === 'object' && storage) return wrap(self, storage) - - if (typeof storage === 'string') { - folder = storage - storage = raf - } - - return { - metadata: function (name) { - return storage(path.join(folder, 'metadata', name)) - }, - content: function (name) { - return storage(path.join(folder, 'content', name)) - } - } -} diff --git a/package.json b/package.json index 58832111..016d6721 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "hyperdrive", - "version": "10.0.0-rc3", + "version": "10.0.0-rc4", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { - "test": "tape test/*.js" + "test": "tape test/index.js" }, "repository": { "type": "git", diff --git a/schema.proto b/schema.proto index 6a89ff2b..b1568e61 100644 --- a/schema.proto +++ b/schema.proto @@ -13,4 +13,5 @@ message Stat { optional uint64 byteOffset = 7; optional uint64 mtime = 8; optional uint64 ctime = 9; + optional string linkname = 10; } diff --git a/test/basic.js b/test/basic.js index 2ee73320..bf5f5931 100644 --- a/test/basic.js +++ b/test/basic.js @@ -51,27 +51,34 @@ tape('write and read (2 parallel)', function (t) { }) }) -tape('write and read (sparse)', function (t) { +tape.only('write and read (sparse)', function (t) { t.plan(2) var archive = create() archive.on('ready', function () { + console.log('CREATING CLONE WITH KEY:', archive.key) var clone = create(archive.key, {sparse: true}) archive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - var stream = clone.replicate() - stream.pipe(archive.replicate()).pipe(stream) - - var readStream = clone.createReadStream('/hello.txt') - readStream.on('data', function (data) { - t.same(data.toString(), 'world') - }) + var s1 = clone.replicate() + var s2 = archive.replicate() + // stream.pipe(archive.replicate()).pipe(stream) + s1.pipe(s2).pipe(s1) + s1.on('feed', dkey => console.log('S1 REPLICATING DKEY:', dkey)) + s1.on('data', d => console.log('S1 DATA:', d)) + s2.on('data', d => console.log('S2 DATA:', d)) + setTimeout(() => { + var readStream = clone.createReadStream('/hello.txt') + readStream.on('data', function (data) { + t.same(data.toString(), 'world') + }) + }, 2000) }) }) }) -tape('write and unlink', function (t) { +tape.skip('write and unlink', function (t) { var archive = create() archive.writeFile('/hello.txt', 'world', function (err) { @@ -86,7 +93,7 @@ tape('write and unlink', function (t) { }) }) -tape('root is always there', function (t) { +tape.skip('root is always there', function (t) { var archive = create() archive.access('/', function (err) { @@ -99,7 +106,7 @@ tape('root is always there', function (t) { }) }) -tape('provide keypair', function (t) { +tape.skip('provide keypair', function (t) { var publicKey = Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES) var secretKey = Buffer.allocUnsafe(sodium.crypto_sign_SECRETKEYBYTES) @@ -124,7 +131,7 @@ tape('provide keypair', function (t) { }) }) -tape('write and read, no cache', function (t) { +tape.skip('write and read, no cache', function (t) { var archive = create({ metadataStorageCacheSize: 0, contentStorageCacheSize: 0, @@ -141,120 +148,7 @@ tape('write and read, no cache', function (t) { }) }) -// TODO: Re-enable the following tests once the `download` and `fetchLatest` APIs are reimplemented. - -tape.skip('download a version', function (t) { - var src = create() - src.on('ready', function () { - t.ok(src.writable) - t.ok(src.metadata.writable) - t.ok(src.content.writable) - src.writeFile('/first.txt', 'number 1', function (err) { - t.error(err, 'no error') - src.writeFile('/second.txt', 'number 2', function (err) { - t.error(err, 'no error') - src.writeFile('/third.txt', 'number 3', function (err) { - t.error(err, 'no error') - t.same(src.version, 3) - testDownloadVersion() - }) - }) - }) - }) - - function testDownloadVersion () { - var clone = create(src.key, { sparse: true }) - clone.on('content', function () { - t.same(clone.version, 3) - clone.checkout(2).download(function (err) { - t.error(err) - clone.readFile('/second.txt', { cached: true }, function (err, content) { - t.error(err, 'block not downloaded') - t.same(content && content.toString(), 'number 2', 'content does not match') - clone.readFile('/third.txt', { cached: true }, function (err, content) { - t.same(err && err.message, 'Block not downloaded') - t.end() - }) - }) - }) - }) - var stream = clone.replicate() - stream.pipe(src.replicate()).pipe(stream) - } -}) - -tape.skip('closing a read-only, latest clone', function (t) { - // This is just a sample key of a dead dat - var clone = create('1d5e5a628d237787afcbfec7041a16f67ba6895e7aa31500013e94ddc638328d', { - latest: true - }) - clone.on('error', function (err) { - t.fail(err) - }) - clone.close(function (err) { - t.error(err) - t.end() - }) -}) - -tape('simple watch', function (t) { - const db = create(null) - - var watchEvents = 0 - db.ready(err => { - t.error(err, 'no error') - db.watch('/a/path/', () => { - if (++watchEvents === 2) { - t.end() - } - }) - doWrites() - }) - - function doWrites () { - db.writeFile('/a/path/hello', 't1', err => { - t.error(err, 'no error') - db.writeFile('/b/path/hello', 't2', err => { - t.error(err, 'no error') - db.writeFile('/a/path/world', 't3', err => { - t.error(err, 'no error') - }) - }) - }) - } -}) - -tape('simple checkout', function (t) { - const drive = create(null) - - drive.writeFile('/hello', 'world', err => { - t.error(err, 'no error') - let version = drive.version - drive.readFile('/hello', (err, data) => { - t.error(err, 'no error') - t.same(data, Buffer.from('world')) - drive.unlink('/hello', err => { - t.error(err, 'no error') - drive.readFile('/hello', (err, data) => { - t.true(err) - t.same(err.code, 'ENOENT') - testCheckout(version) - }) - }) - }) - }) - - function testCheckout (version) { - let oldVersion = drive.checkout(version) - oldVersion.readFile('/hello', (err, data) => { - t.error(err, 'no error') - t.same(data, Buffer.from('world')) - t.end() - }) - } -}) - -tape('can read a single directory', async function (t) { +tape.skip('can read a single directory', async function (t) { const drive = create(null) let files = ['a', 'b', 'c', 'd', 'e', 'f'] @@ -284,7 +178,7 @@ tape('can read a single directory', async function (t) { } }) -tape('can stream a large directory', async function (t) { +tape.skip('can stream a large directory', async function (t) { const drive = create(null) let files = new Array(1000).fill(0).map((_, idx) => '' + idx) @@ -316,7 +210,7 @@ tape('can stream a large directory', async function (t) { } }) -tape('can read nested directories', async function (t) { +tape.skip('can read nested directories', async function (t) { const drive = create(null) let files = ['a', 'b/a/b', 'b/c', 'c/b', 'd/e/f/g/h', 'd/e/a', 'e/a', 'e/b', 'f', 'g'] @@ -362,7 +256,7 @@ tape('can read nested directories', async function (t) { } }) -tape('can read sparse metadata', async function (t) { +tape.skip('can read sparse metadata', async function (t) { const { read, write } = await getTestDrives() let files = ['a', 'b/a/b', 'b/c', 'c/b', 'd/e/f/g/h', 'd/e/a', 'e/a', 'e/b', 'f', 'g'] @@ -405,62 +299,3 @@ tape('can read sparse metadata', async function (t) { }) } }) - -// TODO: Revisit createDiffStream after hypertrie diff stream bug is fixed. -/* -tape.only('simple diff stream', async function (t) { - let drive = create() - - var v1, v2, v3 - let v3Diff = ['del-hello'] - let v2Diff = [...v3Diff, 'put-other'] - let v1Diff = [...v2Diff, 'put-hello'] - - await writeVersions() - console.log('drive.version:', drive.version, 'v1:', v1) - // await verifyDiffStream(v1, v1Diff) - // await verifyDiffStream(v2, v2Diff) - await verifyDiffStream(v3, v3Diff) - t.end() - - function writeVersions () { - return new Promise(resolve => { - drive.ready(err => { - t.error(err, 'no error') - v1 = drive.version - drive.writeFile('/hello', 'world', err => { - t.error(err, 'no error') - v2 = drive.version - drive.writeFile('/other', 'file', err => { - t.error(err, 'no error') - v3 = drive.version - drive.unlink('/hello', err => { - t.error(err, 'no error') - return resolve() - }) - }) - }) - }) - }) - } - - async function verifyDiffStream (version, diffList) { - let diffSet = new Set(diffList) - console.log('diffing to version:', version, 'from version:', drive.version) - let diffStream = drive.createDiffStream(version) - return new Promise(resolve => { - diffStream.on('data', ({ type, name }) => { - let key = `${type}-${name}` - if (!diffSet.has(key)) { - return t.fail('an incorrect diff was streamed') - } - diffSet.delete(key) - }) - diffStream.on('end', () => { - t.same(diffSet.size, 0) - return resolve() - }) - }) - } -}) -*/ diff --git a/test/checkout.js b/test/checkout.js new file mode 100644 index 00000000..b59d80fd --- /dev/null +++ b/test/checkout.js @@ -0,0 +1,89 @@ +var tape = require('tape') +var create = require('./helpers/create') + +tape('simple checkout', function (t) { + const drive = create(null) + + drive.writeFile('/hello', 'world', err => { + t.error(err, 'no error') + let version = drive.version + drive.readFile('/hello', (err, data) => { + t.error(err, 'no error') + t.same(data, Buffer.from('world')) + drive.unlink('/hello', err => { + t.error(err, 'no error') + drive.readFile('/hello', (err, data) => { + t.true(err) + t.same(err.code, 'ENOENT') + testCheckout(version) + }) + }) + }) + }) + + function testCheckout (version) { + let oldVersion = drive.checkout(version) + oldVersion.readFile('/hello', (err, data) => { + t.error(err, 'no error') + t.same(data, Buffer.from('world')) + t.end() + }) + } +}) + +// TODO: Re-enable the following tests once the `download` and `fetchLatest` APIs are reimplemented. + +tape.skip('download a version', function (t) { + var src = create() + src.on('ready', function () { + t.ok(src.writable) + t.ok(src.metadata.writable) + t.ok(src.content.writable) + src.writeFile('/first.txt', 'number 1', function (err) { + t.error(err, 'no error') + src.writeFile('/second.txt', 'number 2', function (err) { + t.error(err, 'no error') + src.writeFile('/third.txt', 'number 3', function (err) { + t.error(err, 'no error') + t.same(src.version, 3) + testDownloadVersion() + }) + }) + }) + }) + + function testDownloadVersion () { + var clone = create(src.key, { sparse: true }) + clone.on('content', function () { + t.same(clone.version, 3) + clone.checkout(2).download(function (err) { + t.error(err) + clone.readFile('/second.txt', { cached: true }, function (err, content) { + t.error(err, 'block not downloaded') + t.same(content && content.toString(), 'number 2', 'content does not match') + clone.readFile('/third.txt', { cached: true }, function (err, content) { + t.same(err && err.message, 'Block not downloaded') + t.end() + }) + }) + }) + }) + var stream = clone.replicate() + stream.pipe(src.replicate()).pipe(stream) + } +}) + +tape.skip('closing a read-only, latest clone', function (t) { + // This is just a sample key of a dead dat + var clone = create('1d5e5a628d237787afcbfec7041a16f67ba6895e7aa31500013e94ddc638328d', { + latest: true + }) + clone.on('error', function (err) { + t.fail(err) + }) + clone.close(function (err) { + t.error(err) + t.end() + }) +}) + diff --git a/test/deletion.js b/test/deletion.js new file mode 100644 index 00000000..7ffc3a5a --- /dev/null +++ b/test/deletion.js @@ -0,0 +1,18 @@ +var tape = require('tape') +var create = require('./helpers/create') + +tape('write and unlink', function (t) { + var archive = create() + + archive.writeFile('/hello.txt', 'world', function (err) { + t.error(err, 'no error') + archive.unlink('/hello.txt', function (err) { + t.error(err, 'no error') + archive.readFile('/hello.txt', function (err) { + t.ok(err, 'had error') + t.end() + }) + }) + }) +}) + diff --git a/test/diff.js b/test/diff.js new file mode 100644 index 00000000..1cc5e3af --- /dev/null +++ b/test/diff.js @@ -0,0 +1,58 @@ +var tape = require('tape') +var create = require('./helpers/create') + +tape('simple diff stream', async function (t) { + let drive = create() + + var v1, v2, v3 + let v3Diff = ['del-hello'] + let v2Diff = [...v3Diff, 'put-other'] + let v1Diff = [...v2Diff, 'put-hello'] + + await writeVersions() + console.log('drive.version:', drive.version, 'v1:', v1) + // await verifyDiffStream(v1, v1Diff) + // await verifyDiffStream(v2, v2Diff) + await verifyDiffStream(v3, v3Diff) + t.end() + + function writeVersions () { + return new Promise(resolve => { + drive.ready(err => { + t.error(err, 'no error') + v1 = drive.version + drive.writeFile('/hello', 'world', err => { + t.error(err, 'no error') + v2 = drive.version + drive.writeFile('/other', 'file', err => { + t.error(err, 'no error') + v3 = drive.version + drive.unlink('/hello', err => { + t.error(err, 'no error') + return resolve() + }) + }) + }) + }) + }) + } + + async function verifyDiffStream (version, diffList) { + let diffSet = new Set(diffList) + console.log('diffing to version:', version, 'from version:', drive.version) + let diffStream = drive.createDiffStream(version) + return new Promise(resolve => { + diffStream.on('data', ({ type, name }) => { + let key = `${type}-${name}` + if (!diffSet.has(key)) { + return t.fail('an incorrect diff was streamed') + } + diffSet.delete(key) + }) + diffStream.on('end', () => { + t.same(diffSet.size, 0) + return resolve() + }) + }) + } +}) diff --git a/test/index.js b/test/index.js new file mode 100644 index 00000000..5440bd80 --- /dev/null +++ b/test/index.js @@ -0,0 +1,10 @@ +require('./basic') +require('./checkout') +require('./creation') +require('./deletion') +// require('./diff') +require('./fd') +require('./fuzzing') +require('./stat') +require('./storage') +// require('./watch') diff --git a/test/watch.js b/test/watch.js new file mode 100644 index 00000000..38883b81 --- /dev/null +++ b/test/watch.js @@ -0,0 +1,30 @@ +var tape = require('tape') +var create = require('./helpers/create') + +tape('simple watch', function (t) { + const db = create(null) + + var watchEvents = 0 + db.ready(err => { + t.error(err, 'no error') + db.watch('/a/path/', () => { + if (++watchEvents === 2) { + t.end() + } + }) + doWrites() + }) + + function doWrites () { + db.writeFile('/a/path/hello', 't1', err => { + t.error(err, 'no error') + db.writeFile('/b/path/hello', 't2', err => { + t.error(err, 'no error') + db.writeFile('/a/path/world', 't3', err => { + t.error(err, 'no error') + }) + }) + }) + } +}) + From b9f967f87b356132fafa7e1675597c1cde60463d Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 8 May 2019 15:31:16 +0200 Subject: [PATCH 063/108] Add temporary git deps --- package.json | 3 ++- test/basic.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 016d6721..c5d14aa5 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ "filesystem-constants": "^1.0.0", "hypercore": "^6.25.0", "hypercore-byte-stream": "^1.0.2", - "hypertrie": "^3.4.0", + "mountable-hypertrie": "git+git@github.com:andrewosh/mountable-hypertrie", + "random-access-corestore": "git+git@github.com:andrewosh/random-access-corestore", "mutexify": "^1.2.0", "pump": "^3.0.0", "pumpify": "^1.5.1", diff --git a/test/basic.js b/test/basic.js index bf5f5931..61360b2c 100644 --- a/test/basic.js +++ b/test/basic.js @@ -69,6 +69,7 @@ tape.only('write and read (sparse)', function (t) { s1.on('data', d => console.log('S1 DATA:', d)) s2.on('data', d => console.log('S2 DATA:', d)) setTimeout(() => { + console.log('S2 METADATA LENGTH:', clone.metadata.length, 'METADATA:', clone.metadata) var readStream = clone.createReadStream('/hello.txt') readStream.on('data', function (data) { t.same(data.toString(), 'world') From 21436d6544f65b14a1bd10d2e0d5e04978ec3916 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 8 May 2019 22:23:51 +0200 Subject: [PATCH 064/108] Basic tests pass --- index.js | 29 ++++------------------------- test/basic.js | 42 +++++++++++------------------------------- 2 files changed, 15 insertions(+), 56 deletions(-) diff --git a/index.js b/index.js index 1c622ca6..87b72143 100644 --- a/index.js +++ b/index.js @@ -54,10 +54,9 @@ class Hyperdrive extends EventEmitter { storageCacheSize: opts.metadataStorageCacheSize, valueEncoding: 'binary' }) - console.log('METADATA:', this.metadata) this._db = opts._db || new MountableHypertrie(this._corestore, key, { feed: this.metadata }) - this._contentFeeds = new WeakMap() + this._contentFeeds = new Map() if (opts.content) this._contentFeeds.set(this._db, new ContentState(opts.content)) this._fds = [] @@ -80,11 +79,10 @@ class Hyperdrive extends EventEmitter { } get writable () { - return this.metadata.writable && this.content.writable + return this.metadata.writable } _ready (cb) { - console.log('IN _READY') const self = this self.metadata.on('error', onerror) @@ -93,8 +91,6 @@ class Hyperdrive extends EventEmitter { return self.metadata.ready(err => { if (err) return cb(err) - console.log('METADATA IS READY') - if (self.sparseMetadata) { self.metadata.update(function loop () { self.metadata.update(loop) @@ -102,11 +98,6 @@ class Hyperdrive extends EventEmitter { } const rootContentKeyPair = self.metadata.secretKey ? contentKeyPair(self.metadata.secretKey) : {} - const rootContentOpts = contentOptions(self, rootContentKeyPair.secretKey) - rootContentOpts.keyPair = rootContentKeyPair - console.log('rootContentOpts:', rootContentOpts) - - console.log('self.metadata:', self.metadata) /** * If a db is provided as input, ensure that a contentFeed is also provided, then return (this is a checkout). @@ -117,7 +108,6 @@ class Hyperdrive extends EventEmitter { * Initialize the db without metadata and load the content feed key from the header. */ if (self.opts._db) { - console.log('A _DB WAS PROVIDED IN OPTS') if (!self.contentFeeds.get(self.opts._db.key)) return cb(new Error('Must provide a db and a content feed')) return done(null) } else if (self.metadata.writable && !self.metadata.length) { @@ -131,7 +121,6 @@ class Hyperdrive extends EventEmitter { * The first time the hyperdrive is created, we initialize both the db (metadata feed) and the content feed here. */ function initialize (keyPair) { - console.log('INITIALIZING') self._db.setMetadata(keyPair.publicKey) self._db.ready(err => { if (err) return cb(err) @@ -148,9 +137,7 @@ class Hyperdrive extends EventEmitter { * (Otherwise, we need to read the feed's metadata block first) */ function restore (keyPair) { - console.log('RESTORING') if (self.metadata.writable) { - console.log('IT IS WRITABLE') self._db.ready(err => { if (err) return done(err) self._getContent(self._db, done) @@ -161,7 +148,6 @@ class Hyperdrive extends EventEmitter { } function done (err) { - console.log('DONE WITH READY, err:', err) if (err) return cb(err) self.key = self.metadata.key self.discoveryKey = self.metadata.discoveryKey @@ -181,23 +167,20 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'function') return this._getContent(db, null, opts) const self = this - console.log('GETTING CONTENT FOR:', 'db:', db) const existingContent = self._contentFeeds.get(db) if (existingContent) return process.nextTick(cb, null, existingContent) - console.log(' NOT EXISTING') db.getMetadata((err, publicKey) => { if (err) return cb(err) - console.log(' METADATA:', publicKey) return onkey(publicKey) }) function onkey (publicKey) { - const feed = self._corestore.get({ key: publicKey, ...self._contentOpts, ...opts }) + const contentOpts = { key: publicKey, ...contentOptions(self, opts && opts.secretKey), ...opts } + const feed = self._corestore.get(contentOpts) feed.ready(err => { if (err) return cb(err) const state = new ContentState(feed, mutexify()) - console.log(' SETTING CONTENT FEED FOR:', db) self._contentFeeds.set(db, state) feed.on('error', err => self.emit('error', err)) return cb(null, state) @@ -259,7 +242,6 @@ class Hyperdrive extends EventEmitter { } createReadStream (name, opts) { - console.log('IN CREATEREADSTREAM FOR:', name) if (!opts) opts = {} name = unixify(name) @@ -274,7 +256,6 @@ class Hyperdrive extends EventEmitter { if (err) return stream.destroy(err) this._db.get(name, (err, node, trie) => { if (err) return stream.destroy(err) - console.log('HERE ERR:', err, 'NODE:', node) if (!node) return stream.destroy(new errors.FileNotFound(name)) this._getContent(trie, (err, contentState) => { @@ -385,7 +366,6 @@ class Hyperdrive extends EventEmitter { if (err) return proxy.destroy(err) this._getContent(trie, (err, contentState) => { if (err) return proxy.destroy(err) - console.log('contentState:', contentState) contentState.lock(_release => { release = _release append(contentState) @@ -430,7 +410,6 @@ class Hyperdrive extends EventEmitter { self._db.put(name, encoded, function (err) { if (err) return proxy.destroy(err) self.emit('append', name, opts) - console.log('AFTER WRite FOR:', name, 'METADATA:', self.metadata) proxy.uncork() }) }) diff --git a/test/basic.js b/test/basic.js index 61360b2c..6e795e40 100644 --- a/test/basic.js +++ b/test/basic.js @@ -51,7 +51,7 @@ tape('write and read (2 parallel)', function (t) { }) }) -tape.only('write and read (sparse)', function (t) { +tape('write and read (sparse)', function (t) { t.plan(2) var archive = create() @@ -61,40 +61,21 @@ tape.only('write and read (sparse)', function (t) { archive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - var s1 = clone.replicate() - var s2 = archive.replicate() + var s1 = clone.replicate({ live: true }) + var s2 = archive.replicate({ live: true }) // stream.pipe(archive.replicate()).pipe(stream) s1.pipe(s2).pipe(s1) - s1.on('feed', dkey => console.log('S1 REPLICATING DKEY:', dkey)) - s1.on('data', d => console.log('S1 DATA:', d)) - s2.on('data', d => console.log('S2 DATA:', d)) setTimeout(() => { - console.log('S2 METADATA LENGTH:', clone.metadata.length, 'METADATA:', clone.metadata) var readStream = clone.createReadStream('/hello.txt') readStream.on('data', function (data) { t.same(data.toString(), 'world') }) - }, 2000) + }, 100) }) }) }) -tape.skip('write and unlink', function (t) { - var archive = create() - - archive.writeFile('/hello.txt', 'world', function (err) { - t.error(err, 'no error') - archive.unlink('/hello.txt', function (err) { - t.error(err, 'no error') - archive.readFile('/hello.txt', function (err) { - t.ok(err, 'had error') - t.end() - }) - }) - }) -}) - -tape.skip('root is always there', function (t) { +tape('root is always there', function (t) { var archive = create() archive.access('/', function (err) { @@ -107,7 +88,7 @@ tape.skip('root is always there', function (t) { }) }) -tape.skip('provide keypair', function (t) { +tape('provide keypair', function (t) { var publicKey = Buffer.allocUnsafe(sodium.crypto_sign_PUBLICKEYBYTES) var secretKey = Buffer.allocUnsafe(sodium.crypto_sign_SECRETKEYBYTES) @@ -118,7 +99,6 @@ tape.skip('provide keypair', function (t) { archive.on('ready', function () { t.ok(archive.writable) t.ok(archive.metadata.writable) - t.ok(archive.content.writable) t.ok(publicKey.equals(archive.key)) archive.writeFile('/hello.txt', 'world', function (err) { @@ -132,7 +112,7 @@ tape.skip('provide keypair', function (t) { }) }) -tape.skip('write and read, no cache', function (t) { +tape('write and read, no cache', function (t) { var archive = create({ metadataStorageCacheSize: 0, contentStorageCacheSize: 0, @@ -149,7 +129,7 @@ tape.skip('write and read, no cache', function (t) { }) }) -tape.skip('can read a single directory', async function (t) { +tape('can read a single directory', async function (t) { const drive = create(null) let files = ['a', 'b', 'c', 'd', 'e', 'f'] @@ -179,7 +159,7 @@ tape.skip('can read a single directory', async function (t) { } }) -tape.skip('can stream a large directory', async function (t) { +tape('can stream a large directory', async function (t) { const drive = create(null) let files = new Array(1000).fill(0).map((_, idx) => '' + idx) @@ -211,7 +191,7 @@ tape.skip('can stream a large directory', async function (t) { } }) -tape.skip('can read nested directories', async function (t) { +tape('can read nested directories', async function (t) { const drive = create(null) let files = ['a', 'b/a/b', 'b/c', 'c/b', 'd/e/f/g/h', 'd/e/a', 'e/a', 'e/b', 'f', 'g'] @@ -257,7 +237,7 @@ tape.skip('can read nested directories', async function (t) { } }) -tape.skip('can read sparse metadata', async function (t) { +tape('can read sparse metadata', async function (t) { const { read, write } = await getTestDrives() let files = ['a', 'b/a/b', 'b/c', 'c/b', 'd/e/f/g/h', 'd/e/a', 'e/a', 'e/b', 'f', 'g'] From bde7cbeeed1735ae5e244dec249dd2b9addfd0bf Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sun, 12 May 2019 22:21:30 +0200 Subject: [PATCH 065/108] Simple read/write from mount works --- index.js | 86 +++++++++++++++++++++--------- lib/fd.js | 41 +++++++------- lib/messages.js | 136 ++++++++++++++++++++++++++++++++++++++++++++++- lib/stat.js | 1 + schema.proto | 7 +++ test/creation.js | 1 - test/fd.js | 10 ++-- test/fuzzing.js | 7 +-- test/index.js | 7 +-- test/storage.js | 34 ++++++------ 10 files changed, 256 insertions(+), 74 deletions(-) diff --git a/index.js b/index.js index 87b72143..09e21dd6 100644 --- a/index.js +++ b/index.js @@ -18,7 +18,7 @@ const createFileDescriptor = require('./lib/fd') const Stat = require('./lib/stat') const errors = require('./lib/errors') const messages = require('./lib/messages') -const defaultCorestore = require('random-access-corestore') +const defaultCorestore = require('./lib/storage') const { contentKeyPair, contentOptions, ContentState } = require('./lib/content') // 20 is arbitrary, just to make the fds > stdio etc @@ -44,7 +44,7 @@ class Hyperdrive extends EventEmitter { this.sparse = !!opts.sparse this.sparseMetadata = !!opts.sparseMetadata - this._corestore = opts.corestore || defaultCorestore(path => storage(path), opts) + this._corestore = defaultCorestore(storage, opts) this.metadata = this._corestore.get({ key, main: true, @@ -56,8 +56,8 @@ class Hyperdrive extends EventEmitter { }) this._db = opts._db || new MountableHypertrie(this._corestore, key, { feed: this.metadata }) - this._contentFeeds = new Map() - if (opts.content) this._contentFeeds.set(this._db, new ContentState(opts.content)) + this._contentStates = new Map() + if (opts.content) this._contentStates.set(this._db, new ContentState(opts.content)) this._fds = [] this._writingFds = new Map() @@ -82,6 +82,12 @@ class Hyperdrive extends EventEmitter { return this.metadata.writable } + get contentWritable () { + const contentState = this._contentStates.get(this._db) + if (!contentState) return false + return contentState.feed.writable + } + _ready (cb) { const self = this @@ -108,7 +114,7 @@ class Hyperdrive extends EventEmitter { * Initialize the db without metadata and load the content feed key from the header. */ if (self.opts._db) { - if (!self.contentFeeds.get(self.opts._db.key)) return cb(new Error('Must provide a db and a content feed')) + if (!self.contentStates.get(self.opts._db.key)) return cb(new Error('Must provide a db and a content feed')) return done(null) } else if (self.metadata.writable && !self.metadata.length) { initialize(rootContentKeyPair) @@ -140,7 +146,7 @@ class Hyperdrive extends EventEmitter { if (self.metadata.writable) { self._db.ready(err => { if (err) return done(err) - self._getContent(self._db, done) + self._getContent(self._db, { secretKey: keyPair.secretKey }, done) }) } else { self._db.ready(done) @@ -167,7 +173,8 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'function') return this._getContent(db, null, opts) const self = this - const existingContent = self._contentFeeds.get(db) + const existingContent = self._contentStates.get(db) + console.log('EXISTING CONTENT:', existingContent) if (existingContent) return process.nextTick(cb, null, existingContent) db.getMetadata((err, publicKey) => { @@ -181,7 +188,7 @@ class Hyperdrive extends EventEmitter { feed.ready(err => { if (err) return cb(err) const state = new ContentState(feed, mutexify()) - self._contentFeeds.set(db, state) + self._contentStates.set(db, state) feed.on('error', err => self.emit('error', err)) return cb(null, state) }) @@ -211,9 +218,12 @@ class Hyperdrive extends EventEmitter { open (name, flags, cb) { name = unixify(name) - createFileDescriptor(this, name, flags, (err, fd) => { + this.ready(err => { if (err) return cb(err) - cb(null, STDIO_CAP + this._fds.push(fd) - 1) + createFileDescriptor(this, name, flags, (err, fd) => { + if (err) return cb(err) + cb(null, STDIO_CAP + this._fds.push(fd) - 1) + }) }) } @@ -256,6 +266,7 @@ class Hyperdrive extends EventEmitter { if (err) return stream.destroy(err) this._db.get(name, (err, node, trie) => { if (err) return stream.destroy(err) + console.log('IN CREATEREADSTREAM, DB GET node:', node) if (!node) return stream.destroy(new errors.FileNotFound(name)) this._getContent(trie, (err, contentState) => { @@ -502,14 +513,7 @@ class Hyperdrive extends EventEmitter { }) } - mkdir (name, opts, cb) { - if (typeof opts === 'function') return this.mkdir(name, null, opts) - if (typeof opts === 'number') opts = {mode: opts} - if (!opts) opts = {} - if (!cb) cb = noop - - name = unixify(name) - + _createDirectoryStat (name, opts, cb) { this._db.get(name, (err, node, trie) => { if (err) return cb(err) this._getContent(trie, (err, contentState) => { @@ -517,12 +521,14 @@ class Hyperdrive extends EventEmitter { try { var st = messages.Stat.encode(Stat.directory({ ...opts, - offset: contentState.length, - byteOffset: contentState.byteLength + offset: contentState.feed.length, + byteOffset: contentState.feed.byteLength })) + console.log('CREATING DIR WITH OFFSET:', contentState.feed.length, 'byteOffset:', contentState.feed.byteLength) } catch (err) { return cb(err) } + return cb(null, st) this._db.put(name, st, { condition: ifNotExists }, cb) @@ -530,6 +536,22 @@ class Hyperdrive extends EventEmitter { }) } + mkdir (name, opts, cb) { + if (typeof opts === 'function') return this.mkdir(name, null, opts) + if (typeof opts === 'number') opts = {mode: opts} + if (!opts) opts = {} + if (!cb) cb = noop + + name = unixify(name) + + this._createDirectoryStat(name, opts, (err, st) => { + if (err) return cb(err) + this._db.put(name, st, { + condition: ifNotExists + }, cb) + }) + } + _statDirectory (name, opts, cb) { const ite = this._db.iterator(name) ite.next((err, st) => { @@ -670,13 +692,14 @@ class Hyperdrive extends EventEmitter { if (typeof fd === 'number') return this._closeFile(fd, cb || noop) else cb = fd if (!cb) cb = noop + const self = this + + // Attempt to close all feeds, even if a subset of them fail. Return the last error. + var closeErr = null this.ready(err => { if (err) return cb(err) - this.metadata.close(err => { - if (!this.content) return cb(err) - this.content.close(cb) - }) + return this._corestore.close(cb) }) } @@ -686,7 +709,20 @@ class Hyperdrive extends EventEmitter { } mount (path, key, opts, cb) { - return this._db.mount(path, key, opts, cb) + if (typeof opts === 'function') return this.mount(path, key, null, opts) + opts = opts || {} + opts.mount = { + key, + version: opts.version, + hash: opts.hash + } + this._createDirectoryStat(path, opts, (err, st) => { + if (err) return cb(err) + this._db.mount(path, key, { ...opts, value: st }, err => { + if (err) return cb(err) + return this._db.loadMount(path, cb) + }) + }) } } diff --git a/lib/fd.js b/lib/fd.js index 24705116..374eed50 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -19,10 +19,11 @@ const { } = linuxConstants class FileDescriptor { - constructor (drive, path, stat, readable, writable, appending, creating) { + constructor (drive, path, stat, contentState, readable, writable, appending, creating) { this.drive = drive this.stat = stat this.path = path + this.contentState = contentState this.readable = readable this.writable = writable @@ -128,7 +129,7 @@ class FileDescriptor { } } if (this._range) { - this.drive.content.undownload(this._range) + this.contentState.feed.undownload(this._range) this._range = null } process.nextTick(cb, null) @@ -143,17 +144,17 @@ class FileDescriptor { const end = Math.min(this.stat.blocks + this.stat.offset, start + 16) if (this._range) { - this.drive.content.undownload(this._range) + this.contentState.feed.undownload(this._range) } - this._range = this.drive.content.download({ start, end, linear: true }, cb || noop) + this._range = this.contentState.feed.download({ start, end, linear: true }, cb || noop) } _seekAndRead (buffer, offset, len, pos, cb) { const start = this.stat.offset const end = start + this.stat.blocks - this.drive.content.seek(this.stat.byteOffset + pos, { start, end }, (err, blk, blockOffset) => { + this.contentState.feed.seek(this.stat.byteOffset + pos, { start, end }, (err, blk, blockOffset) => { if (err) return cb(err) this.position = pos this.blockPosition = blk @@ -200,7 +201,7 @@ class FileDescriptor { return process.nextTick(cb, null, 0, buffer) } - this.drive.content.get(blk, (err, data) => { + this.contentState.feed.get(blk, (err, data) => { if (err) return cb(err) if (blkOffset) data = data.slice(blkOffset) @@ -236,22 +237,24 @@ module.exports = function create (drive, name, flags, cb) { const creating = !!(flags & O_CREAT) const canExist = !(flags & O_EXCL) - drive.contentReady(err => { + drive._db.get(name, (err, st, trie) => { if (err) return cb(err) - drive._db.get(name, (err, st) => { - if (err) return cb(err) - if (st && !canExist) return cb(new errors.PathAlreadyExists(name)) - if (!st && (!writable || !creating)) return cb(new errors.FileNotFound(name)) - - if (st) { - try { - st = messages.Stat.decode(st.value) - } catch (err) { - return cb(err) - } + console.log('name:', name, 'st:', st, 'trie:', trie) + if (st && !canExist) return cb(new errors.PathAlreadyExists(name)) + if (!st && (!writable || !creating)) return cb(new errors.FileNotFound(name)) + + if (st) { + try { + st = messages.Stat.decode(st.value) + } catch (err) { + return cb(err) } + } - const fd = new FileDescriptor(drive, name, st, readable, writable, appending, creating) + drive._getContent(trie, (err, contentState) => { + if (err) return cb(err) + console.log('CONTENT STATE HERE:', contentState) + const fd = new FileDescriptor(drive, name, st, contentState, readable, writable, appending, creating) if (creating) { drive.create(name, (err, st) => { if (err) return cb(err) diff --git a/lib/messages.js b/lib/messages.js index 9fc819d2..caadc96b 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -17,6 +17,13 @@ var Index = exports.Index = { decode: null } +var Mount = exports.Mount = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + var Stat = exports.Stat = { buffer: true, encodingLength: null, @@ -25,6 +32,7 @@ var Stat = exports.Stat = { } defineIndex() +defineMount() defineStat() function defineIndex () { @@ -102,11 +110,102 @@ function defineIndex () { } } -function defineStat () { +function defineMount () { var enc = [ + encodings.bytes, encodings.varint ] + Mount.encodingLength = encodingLength + Mount.encode = encode + Mount.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.key)) throw new Error("key is required") + var len = enc[0].encodingLength(obj.key) + length += 1 + len + if (defined(obj.version)) { + var len = enc[1].encodingLength(obj.version) + length += 1 + len + } + if (defined(obj.hash)) { + var len = enc[0].encodingLength(obj.hash) + length += 1 + len + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.key)) throw new Error("key is required") + buf[offset++] = 10 + enc[0].encode(obj.key, buf, offset) + offset += enc[0].encode.bytes + if (defined(obj.version)) { + buf[offset++] = 16 + enc[1].encode(obj.version, buf, offset) + offset += enc[1].encode.bytes + } + if (defined(obj.hash)) { + buf[offset++] = 26 + enc[0].encode(obj.hash, buf, offset) + offset += enc[0].encode.bytes + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + key: null, + version: 0, + hash: null + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.key = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + found0 = true + break + case 2: + obj.version = enc[1].decode(buf, offset) + offset += enc[1].decode.bytes + break + case 3: + obj.hash = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defineStat () { + var enc = [ + encodings.varint, + encodings.string, + Mount + ] + Stat.encodingLength = encodingLength Stat.encode = encode Stat.decode = decode @@ -148,6 +247,15 @@ function defineStat () { var len = enc[0].encodingLength(obj.ctime) length += 1 + len } + if (defined(obj.linkname)) { + var len = enc[1].encodingLength(obj.linkname) + length += 1 + len + } + if (defined(obj.mount)) { + var len = enc[2].encodingLength(obj.mount) + length += varint.encodingLength(len) + length += 1 + len + } return length } @@ -199,6 +307,18 @@ function defineStat () { enc[0].encode(obj.ctime, buf, offset) offset += enc[0].encode.bytes } + if (defined(obj.linkname)) { + buf[offset++] = 82 + enc[1].encode(obj.linkname, buf, offset) + offset += enc[1].encode.bytes + } + if (defined(obj.mount)) { + buf[offset++] = 90 + varint.encode(enc[2].encodingLength(obj.mount), buf, offset) + offset += varint.encode.bytes + enc[2].encode(obj.mount, buf, offset) + offset += enc[2].encode.bytes + } encode.bytes = offset - oldOffset return buf } @@ -217,7 +337,9 @@ function defineStat () { offset: 0, byteOffset: 0, mtime: 0, - ctime: 0 + ctime: 0, + linkname: "", + mount: null } var found0 = false while (true) { @@ -267,6 +389,16 @@ function defineStat () { obj.ctime = enc[0].decode(buf, offset) offset += enc[0].decode.bytes break + case 10: + obj.linkname = enc[1].decode(buf, offset) + offset += enc[1].decode.bytes + break + case 11: + var len = varint.decode(buf, offset) + offset += varint.decode.bytes + obj.mount = enc[2].decode(buf, offset, offset + len) + offset += enc[2].decode.bytes + break default: offset = skip(prefix & 7, buf, offset) } diff --git a/lib/stat.js b/lib/stat.js index 79ee1588..19bc29c8 100644 --- a/lib/stat.js +++ b/lib/stat.js @@ -21,6 +21,7 @@ class Stat { this.mtime = data && data.mtime ? getTime(data.mtime) : Date.now() this.ctime = data && data.ctime ? getTime(data.ctime) : Date.now() this.linkname = (data && data.linkname) || null + this.mount = (data && data.mount) || null } _check (mask) { diff --git a/schema.proto b/schema.proto index b1568e61..78ff0fe0 100644 --- a/schema.proto +++ b/schema.proto @@ -3,6 +3,12 @@ message Index { optional bytes content = 2; } +message Mount { + required bytes key = 1; + optional uint64 version = 2; + optional bytes hash = 3; +} + message Stat { required uint32 mode = 1; optional uint32 uid = 2; @@ -14,4 +20,5 @@ message Stat { optional uint64 mtime = 8; optional uint64 ctime = 9; optional string linkname = 10; + optional Mount mount = 11; } diff --git a/test/creation.js b/test/creation.js index 15c861f2..3f6f927f 100644 --- a/test/creation.js +++ b/test/creation.js @@ -7,7 +7,6 @@ tape('owner is writable', function (t) { archive.on('ready', function () { t.ok(archive.writable) t.ok(archive.metadata.writable) - t.ok(archive.content.writable) t.end() }) }) diff --git a/test/fd.js b/test/fd.js index 4f87a3d8..e00dbeb4 100644 --- a/test/fd.js +++ b/test/fd.js @@ -195,7 +195,7 @@ tape('fd basic write, creating file', function (t) { }) }) -tape('fd basic write, appending file', function (t) { +tape.skip('fd basic write, appending file', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) @@ -225,7 +225,7 @@ tape('fd basic write, appending file', function (t) { } }) -tape('fd basic write, overwrite file', function (t) { +tape.skip('fd basic write, overwrite file', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) @@ -255,7 +255,7 @@ tape('fd basic write, overwrite file', function (t) { } }) -tape('fd stateful write', function (t) { +tape.skip('fd stateful write', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) @@ -280,7 +280,7 @@ tape('fd stateful write', function (t) { }) }) -tape('huge stateful write + stateless read', function (t) { +tape.skip('huge stateful write + stateless read', function (t) { const NUM_SLICES = 1000 const SLICE_SIZE = 4096 const READ_SIZE = Math.floor(4096 * 2.75) @@ -336,7 +336,7 @@ tape('huge stateful write + stateless read', function (t) { } }) -tape('fd random-access write fails', function (t) { +tape.skip('fd random-access write fails', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) diff --git a/test/fuzzing.js b/test/fuzzing.js index c4578c16..54fe9bc9 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -91,7 +91,7 @@ class HyperdriveFuzzer extends FuzzBuzz { this.log = [] return new Promise((resolve, reject) => { - this.drive.contentReady(err => { + this.drive.ready(err => { if (err) return reject(err) return resolve() }) @@ -206,6 +206,7 @@ class HyperdriveFuzzer extends FuzzBuzz { if (err) return reject(err) if (!st) return reject(new Error(`Directory ${dirName} should exist but does not exist.`)) if (!st.isDirectory()) return reject(new Error(`Stat for directory ${dirName} does not have directory mode`)) + console.log('st:', st, 'offset:', offset, 'byteOffset:', byteOffset) if (st.offset !== offset || st.byteOffset !== byteOffset) return reject(new Error(`Invalid offsets for ${dirName}`)) this.debug(` Successfully statted directory.`) return resolve({ type: 'stat', dirName }) @@ -457,7 +458,7 @@ class SparseHyperdriveFuzzer extends HyperdriveFuzzer { if (err) throw err let s1 = this.remoteDrive.replicate({ live: true }) s1.pipe(this.drive.replicate({ live: true })).pipe(s1) - this.remoteDrive.contentReady(err => { + this.remoteDrive.ready(err => { if (err) return reject(err) return resolve() }) @@ -471,7 +472,7 @@ class SparseHyperdriveFuzzer extends HyperdriveFuzzer { module.exports = HyperdriveFuzzer -tape('20000 mixed operations, single drive', async t => { +tape.only('20000 mixed operations, single drive', async t => { t.plan(1) const fuzz = new HyperdriveFuzzer({ diff --git a/test/index.js b/test/index.js index 5440bd80..888f61c5 100644 --- a/test/index.js +++ b/test/index.js @@ -1,10 +1,11 @@ require('./basic') -require('./checkout') +//require('./checkout') require('./creation') require('./deletion') // require('./diff') require('./fd') -require('./fuzzing') +// require('./fuzzing') require('./stat') require('./storage') -// require('./watch') +require('./watch') +require('./mount') diff --git a/test/storage.js b/test/storage.js index 11542ce1..80f72382 100644 --- a/test/storage.js +++ b/test/storage.js @@ -8,7 +8,7 @@ tape('ram storage', function (t) { archive.ready(function () { t.ok(archive.metadata.writable, 'archive metadata is writable') - t.ok(archive.content.writable, 'archive content is writable') + t.ok(archive.contentWritable, 'archive content is writable') t.end() }) }) @@ -19,7 +19,7 @@ tape('dir storage with resume', function (t) { var archive = hyperdrive(dir) archive.ready(function () { t.ok(archive.metadata.writable, 'archive metadata is writable') - t.ok(archive.content.writable, 'archive content is writable') + t.ok(archive.contentWritable, 'archive content is writable') t.same(archive.version, 1, 'archive has version 1') archive.close(function (err) { t.ifError(err) @@ -28,7 +28,7 @@ tape('dir storage with resume', function (t) { archive2.ready(function (err) { t.error(err, 'no error') t.ok(archive2.metadata.writable, 'archive2 metadata is writable') - t.ok(archive2.content.writable, 'archive2 content is writable') + t.ok(archive2.contentWritable, 'archive2 content is writable') t.same(archive2.version, 1, 'archive has version 1') cleanup(function (err) { @@ -48,9 +48,9 @@ tape('dir storage for non-writable archive', function (t) { t.ifError(err) var clone = hyperdrive(dir, src.key) - clone.on('content', function () { + clone.ready(function () { t.ok(!clone.metadata.writable, 'clone metadata not writable') - t.ok(!clone.content.writable, 'clone content not writable') + t.ok(!clone.contentWritable, 'clone content not writable') t.same(clone.key, src.key, 'keys match') cleanup(function (err) { t.ifError(err) @@ -83,15 +83,17 @@ tape('write and read (sparse)', function (t) { clone.on('ready', function () { archive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - var stream = clone.replicate() - stream.pipe(archive.replicate()).pipe(stream) - var readStream = clone.createReadStream('/hello.txt') - readStream.on('error', function (err) { - t.error(err, 'no error') - }) - readStream.on('data', function (data) { - t.same(data.toString(), 'world') - }) + var stream = clone.replicate({ live: true }) + stream.pipe(archive.replicate({ live: true })).pipe(stream) + setTimeout(() => { + var readStream = clone.createReadStream('/hello.txt') + readStream.on('error', function (err) { + t.error(err, 'no error') + }) + readStream.on('data', function (data) { + t.same(data.toString(), 'world') + }) + }, 50) }) }) }) @@ -106,8 +108,8 @@ tape('sparse read/write two files', function (t) { t.error(err, 'no error') archive.writeFile('/hello2.txt', 'world', function (err) { t.error(err, 'no error') - var stream = clone.replicate() - stream.pipe(archive.replicate()).pipe(stream) + var stream = clone.replicate({ live: true }) + stream.pipe(archive.replicate({ live: true })).pipe(stream) clone.metadata.update(start) }) }) From 49f2c28260e3dc069e73997dd3fdbb8becdf3e80 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 13 May 2019 00:10:51 +0200 Subject: [PATCH 066/108] Added mount tests --- lib/storage.js | 16 +++++++++++ test/mount.js | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 lib/storage.js create mode 100644 test/mount.js diff --git a/lib/storage.js b/lib/storage.js new file mode 100644 index 00000000..de44fba3 --- /dev/null +++ b/lib/storage.js @@ -0,0 +1,16 @@ +const raf = require('random-access-file') +const corestore = require('random-access-corestore') + +module.exports = function defaultCorestore (storage, opts) { + if (isCorestore(storage)) return storage + if (typeof storage === 'function') { + var factory = path => storage(path) + } else if (typeof storage === 'string') { + factory = path => raf(storage + '/' + path) + } + return corestore(factory, opts) +} + +function isCorestore (storage) { + return storage.get && storage.replicate && storage.close +} diff --git a/test/mount.js b/test/mount.js new file mode 100644 index 00000000..45dc70c7 --- /dev/null +++ b/test/mount.js @@ -0,0 +1,74 @@ +var test = require('tape') +var create = require('./helpers/create') + +test('basic read/write to/from a mount', t => { + const archive1 = create() + const archive2 = create() + + const s1 = archive1.replicate({ live: true, encrypt: false }) + s1.pipe(archive2.replicate({ live: true, encrypt: false })).pipe(s1) + + archive2.ready(err => { + t.error(err, 'no error') + archive2.writeFile('b', 'hello', err => { + t.error(err, 'no error') + archive1.mount('a', archive2.key, err => { + t.error(err, 'no error') + archive1.readFile('a/b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + t.end() + }) + }) + }) + }) +}) + +test('readdir returns mounts', t => { + const archive1 = create() + const archive2 = create() + + const s1 = archive1.replicate({ live: true, encrypt: false }) + s1.pipe(archive2.replicate({ live: true, encrypt: false })).pipe(s1) + + archive2.ready(err => { + t.error(err, 'no error') + archive1.mkdir('b', err => { + t.error(err, 'no error') + archive1.mkdir('b/a', err => { + t.error(err, 'no error') + archive1.mount('a', archive2.key, err => { + t.error(err, 'no error') + archive1.readdir('/', (err, dirs) => { + t.error(err, 'no error') + t.same(dirs, ['b', 'a']) + t.end() + }) + }) + }) + }) + }) +}) + +test('cross-mount watch', t => { + const archive1 = create() + const archive2 = create() + + const s1 = archive1.replicate({ live: true, encrypt: false }) + s1.pipe(archive2.replicate({ live: true, encrypt: false })).pipe(s1) + + var watchEvents = 0 + + archive2.ready(err => { + t.error(err, 'no error') + archive1.mount('a', archive2.key, err => { + t.error(err, 'no error') + archive1.watch('/', () => { + if (++watchEvents === 1) t.end() + }) + archive2.writeFile('a', 'hello', err => { + t.error(err, 'no error') + }) + }) + }) +}) From f7cf4f31f38b31158ee74e26db2c059fc65d98a4 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 16 May 2019 10:44:54 +0200 Subject: [PATCH 067/108] Added symlinking + tests --- index.js | 139 +++++++++++++++++++++++++------------------------- lib/fd.js | 14 +---- lib/stat.js | 5 ++ test/mount.js | 30 +++++++++++ 4 files changed, 107 insertions(+), 81 deletions(-) diff --git a/index.js b/index.js index 09e21dd6..4142388b 100644 --- a/index.js +++ b/index.js @@ -195,6 +195,19 @@ class Hyperdrive extends EventEmitter { } } + _putStat (name, stat, opts, cb) { + if (typeof opts === 'function') return this._putStat(name, stat, null, opts) + try { + var encoded = messages.Stat.encode(stat) + } catch (err) { + return cb(err) + } + this._db.put(name, encoded, opts, err => { + if (err) return cb(err) + return cb(null, stat) + }) + } + _update (name, stat, cb) { name = unixify(name) @@ -203,15 +216,11 @@ class Hyperdrive extends EventEmitter { if (!st) return cb(new errors.FileNotFound(name)) try { var decoded = messages.Stat.decode(st.value) - const newStat = Object.assign(decoded, stat) - var encoded = messages.Stat.encode(newStat) } catch (err) { return cb(err) } - this._db.put(name, encoded, err => { - if (err) return cb(err) - return cb(null) - }) + const newStat = Object.assign(decoded, stat) + return this._putStat(name, newStat, cb) }) } @@ -264,25 +273,16 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return stream.destroy(err) - this._db.get(name, (err, node, trie) => { + this.stat(name, (err, st, trie) => { if (err) return stream.destroy(err) - console.log('IN CREATEREADSTREAM, DB GET node:', node) - if (!node) return stream.destroy(new errors.FileNotFound(name)) - this._getContent(trie, (err, contentState) => { if (err) return stream.destroy(err) - return oncontent(node.value, contentState) + return oncontent(st, contentState) }) }) }) function oncontent (st, contentState) { - try { - st = messages.Stat.decode(st) - } catch (err) { - return stream.destroy(err) - } - const byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) @@ -373,8 +373,8 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return proxy.destroy(err) - this._db.get(name, (err, node, trie) => { - if (err) return proxy.destroy(err) + this.stat(name, { trie: true }, (err, stat, trie) => { + if (err && (err.errno !== 2)) return proxy.destroy(err) this._getContent(trie, (err, contentState) => { if (err) return proxy.destroy(err) contentState.lock(_release => { @@ -410,15 +410,8 @@ class Hyperdrive extends EventEmitter { size: contentState.feed.byteLength - byteOffset, blocks: contentState.feed.length - offset }) - - try { - var encoded = messages.Stat.encode(stat) - } catch (err) { - return proxy.destroy(err) - } - proxy.cork() - self._db.put(name, encoded, function (err) { + self._putStat(name, stat, function (err) { if (err) return proxy.destroy(err) self.emit('append', name, opts) proxy.uncork() @@ -443,16 +436,8 @@ class Hyperdrive extends EventEmitter { this.lstat(name, { file: true }, (err, stat) => { if (err && err.errno !== 2) return cb(err) if (stat) return cb(null, stat) - try { - var st = Stat.file(opts) - var buf = messages.Stat.encode(st) - } catch (err) { - return cb(err) - } - this._db.put(name, buf, err => { - if (err) return cb(err) - return cb(null, st) - }) + const st = Stat.file(opts) + return this._putStat(name, st, cb) }) }) } @@ -518,20 +503,12 @@ class Hyperdrive extends EventEmitter { if (err) return cb(err) this._getContent(trie, (err, contentState) => { if (err) return cb(err) - try { - var st = messages.Stat.encode(Stat.directory({ - ...opts, - offset: contentState.feed.length, - byteOffset: contentState.feed.byteLength - })) - console.log('CREATING DIR WITH OFFSET:', contentState.feed.length, 'byteOffset:', contentState.feed.byteLength) - } catch (err) { - return cb(err) - } + const st = Stat.directory({ + ...opts, + offset: contentState.feed.length, + byteOffset: contentState.feed.byteLength + }) return cb(null, st) - this._db.put(name, st, { - condition: ifNotExists - }, cb) }) }) } @@ -546,7 +523,7 @@ class Hyperdrive extends EventEmitter { this._createDirectoryStat(name, opts, (err, st) => { if (err) return cb(err) - this._db.put(name, st, { + this._putStat(name, st, { condition: ifNotExists }, cb) }) @@ -570,34 +547,42 @@ class Hyperdrive extends EventEmitter { lstat (name, opts, cb) { if (typeof opts === 'function') return this.lstat(name, null, opts) if (!opts) opts = {} + const self = this name = unixify(name) this.ready(err => { if (err) return cb(err) - - this._db.get(name, opts, (err, node) => { - if (err) return cb(err) - if (!node && opts.file) return cb(new errors.FileNotFound(name)) - if (!node) return this._statDirectory(name, opts, cb) - try { - var st = messages.Stat.decode(node.value) - } catch (err) { - return cb(err) - } - const writingFd = this._writingFds.get(name) - if (writingFd) { - st.size = writingFd.stat.size - } - cb(null, new Stat(st)) - }) + this._db.get(name, opts, onstat) }) + + function onstat (err, node, trie) { + if (err) return cb(err) + if (!node && opts.trie) return cb(null, null, trie) + if (!node && opts.file) return cb(new errors.FileNotFound(name)) + if (!node) return self._statDirectory(name, opts, cb) + try { + var st = messages.Stat.decode(node.value) + } catch (err) { + return cb(err) + } + const writingFd = self._writingFds.get(name) + if (writingFd) { + st.size = writingFd.stat.size + } + cb(null, new Stat(st), trie) + } } stat (name, opts, cb) { if (typeof opts === 'function') return this.stat(name, null, opts) if (!opts) opts = {} - this.lstat(name, opts, cb) + this.lstat(name, opts, (err, stat, trie) => { + if (err) return cb(err) + if (!stat) return cb(null, null, trie) + if (stat.linkname) return this.lstat(stat.linkname, opts, cb) + return cb(null, stat, trie) + }) } access (name, opts, cb) { @@ -710,7 +695,9 @@ class Hyperdrive extends EventEmitter { mount (path, key, opts, cb) { if (typeof opts === 'function') return this.mount(path, key, null, opts) + path = unixify(path) opts = opts || {} + opts.mount = { key, version: opts.version, @@ -718,12 +705,26 @@ class Hyperdrive extends EventEmitter { } this._createDirectoryStat(path, opts, (err, st) => { if (err) return cb(err) - this._db.mount(path, key, { ...opts, value: st }, err => { + this._db.mount(path, key, { ...opts, value: messages.Stat.encode(st) }, err => { if (err) return cb(err) return this._db.loadMount(path, cb) }) }) } + + symlink (target, linkName, cb) { + target = unixify(target) + linkName = unixify(linkName) + + this.lstat(linkName, (err, stat) => { + if (err && (err.errno !== 2)) return cb(err) + if (!err) return cb(new errors.PathAlreadyExists(linkName)) + const st = Stat.symlink({ + linkname: target + }) + return this._putStat(linkName, st, cb) + }) + } } function isObject (val) { diff --git a/lib/fd.js b/lib/fd.js index 374eed50..cc38330c 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -237,23 +237,13 @@ module.exports = function create (drive, name, flags, cb) { const creating = !!(flags & O_CREAT) const canExist = !(flags & O_EXCL) - drive._db.get(name, (err, st, trie) => { - if (err) return cb(err) - console.log('name:', name, 'st:', st, 'trie:', trie) + drive.stat(name, { trie: true }, (err, st, trie) => { + if (err && (err.errno !== 2)) return cb(err) if (st && !canExist) return cb(new errors.PathAlreadyExists(name)) if (!st && (!writable || !creating)) return cb(new errors.FileNotFound(name)) - if (st) { - try { - st = messages.Stat.decode(st.value) - } catch (err) { - return cb(err) - } - } - drive._getContent(trie, (err, contentState) => { if (err) return cb(err) - console.log('CONTENT STATE HERE:', contentState) const fd = new FileDescriptor(drive, name, st, contentState, readable, writable, appending, creating) if (creating) { drive.create(name, (err, st) => { diff --git a/lib/stat.js b/lib/stat.js index 19bc29c8..997f4153 100644 --- a/lib/stat.js +++ b/lib/stat.js @@ -61,6 +61,11 @@ Stat.directory = function (data) { data.mode = (data.mode || DEFAULT_DMODE) | Stat.IFDIR return new Stat(data) } +Stat.symlink = function (data) { + data = data || {} + data.mode = (data.mode || DEFAULT_FMODE) | Stat.IFLNK + return new Stat(data) +} Stat.IFSOCK = 0b1100 << 12 Stat.IFLNK = 0b1010 << 12 diff --git a/test/mount.js b/test/mount.js index 45dc70c7..9e94237d 100644 --- a/test/mount.js +++ b/test/mount.js @@ -72,3 +72,33 @@ test('cross-mount watch', t => { }) }) }) + +test('cross-mount symlink', t => { + const archive1 = create() + const archive2 = create() + + const s1 = archive1.replicate({ live: true, encrypt: false }) + s1.pipe(archive2.replicate({ live: true, encrypt: false })).pipe(s1) + + archive2.ready(err => { + t.error(err, 'no error') + archive1.mount('a', archive2.key, err => { + t.error(err, 'no error') + onmount() + }) + }) + + function onmount () { + archive2.writeFile('b', 'hello world', err => { + t.error(err, 'no error') + archive1.symlink('a/b', 'c', err => { + t.error(err, 'no error') + archive1.readFile('c', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello world')) + t.end() + }) + }) + }) + } +}) From e6d4bf9db6859eb6a0928901b90b47c19bd93c10 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 24 May 2019 15:33:11 +0200 Subject: [PATCH 068/108] Update deps + remove console.logs --- index.js | 1 - package.json | 4 ++-- test/basic.js | 2 -- test/mount.js | 3 +++ 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 4142388b..a9f9e9d5 100644 --- a/index.js +++ b/index.js @@ -174,7 +174,6 @@ class Hyperdrive extends EventEmitter { const self = this const existingContent = self._contentStates.get(db) - console.log('EXISTING CONTENT:', existingContent) if (existingContent) return process.nextTick(cb, null, existingContent) db.getMetadata((err, publicKey) => { diff --git a/package.json b/package.json index c5d14aa5..62f5aeae 100644 --- a/package.json +++ b/package.json @@ -27,8 +27,8 @@ "filesystem-constants": "^1.0.0", "hypercore": "^6.25.0", "hypercore-byte-stream": "^1.0.2", - "mountable-hypertrie": "git+git@github.com:andrewosh/mountable-hypertrie", - "random-access-corestore": "git+git@github.com:andrewosh/random-access-corestore", + "mountable-hypertrie": "^0.9.0", + "random-access-corestore": "^0.9.0", "mutexify": "^1.2.0", "pump": "^3.0.0", "pumpify": "^1.5.1", diff --git a/test/basic.js b/test/basic.js index 6e795e40..d167b762 100644 --- a/test/basic.js +++ b/test/basic.js @@ -7,7 +7,6 @@ tape('write and read', function (t) { archive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - console.log('reading') archive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') t.same(buf, Buffer.from('world')) @@ -56,7 +55,6 @@ tape('write and read (sparse)', function (t) { var archive = create() archive.on('ready', function () { - console.log('CREATING CLONE WITH KEY:', archive.key) var clone = create(archive.key, {sparse: true}) archive.writeFile('/hello.txt', 'world', function (err) { diff --git a/test/mount.js b/test/mount.js index 9e94237d..2dd970c0 100644 --- a/test/mount.js +++ b/test/mount.js @@ -102,3 +102,6 @@ test('cross-mount symlink', t => { }) } }) + +test('versioned mount') +test('watch will unwatch on umount') From 41d52c0566cba8432040d221ec72e365bb2b7647 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sun, 9 Jun 2019 04:03:12 +0200 Subject: [PATCH 069/108] Test refactoring + added support for mounting hypercores --- index.js | 92 +++++++++++++++++++------ lib/fd.js | 2 +- lib/storage.js | 2 +- package.json | 1 + schema.proto | 3 + test/helpers/create.js | 6 +- test/mount.js | 150 +++++++++++++++++++++++++++++++---------- 7 files changed, 200 insertions(+), 56 deletions(-) diff --git a/index.js b/index.js index a9f9e9d5..eca9f2e6 100644 --- a/index.js +++ b/index.js @@ -17,8 +17,8 @@ const MountableHypertrie = require('mountable-hypertrie') const createFileDescriptor = require('./lib/fd') const Stat = require('./lib/stat') const errors = require('./lib/errors') -const messages = require('./lib/messages') const defaultCorestore = require('./lib/storage') +const { messages } = require('hyperdrive-schema') const { contentKeyPair, contentOptions, ContentState } = require('./lib/content') // 20 is arbitrary, just to make the fds > stdio etc @@ -45,9 +45,8 @@ class Hyperdrive extends EventEmitter { this.sparseMetadata = !!opts.sparseMetadata this._corestore = defaultCorestore(storage, opts) - this.metadata = this._corestore.get({ + this.metadata = this._corestore.default({ key, - main: true, secretKey: opts.secretKey, sparse: this.sparseMetadata, createIfMissing: opts.createIfMissing, @@ -261,6 +260,7 @@ class Hyperdrive extends EventEmitter { createReadStream (name, opts) { if (!opts) opts = {} + const self = this name = unixify(name) @@ -282,13 +282,26 @@ class Hyperdrive extends EventEmitter { }) function oncontent (st, contentState) { - const byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset + if (st.mount && st.mount.hypercore) { + var byteOffset = 0 + var blockOffset = 0 + var blockLength = st.blocks + var feed = self._corestore.get({ + key: st.mount.key, + sparse: self.sparse + }) + } else { + blockOffset = st.offset + blockLength = st.blocks + byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset + feed = contentState.feed + } const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) stream.start({ - feed: contentState.feed, - blockOffset: st.offset, - blockLength: st.blocks, + feed, + blockOffset, + blockLength, byteOffset, byteLength }) @@ -457,7 +470,7 @@ class Hyperdrive extends EventEmitter { writeFile (name, buf, opts, cb) { if (typeof opts === 'function') return this.writeFile(name, buf, null, opts) - if (typeof opts === 'string') opts = {encoding: opts} + if (typeof opts === 'string') opts = { encoding: opts } if (!opts) opts = {} if (typeof buf === 'string') buf = Buffer.from(buf, opts.encoding || 'utf-8') if (!cb) cb = noop @@ -465,8 +478,17 @@ class Hyperdrive extends EventEmitter { name = unixify(name) let stream = this.createWriteStream(name, opts) - stream.on('error', cb) - stream.on('finish', cb) + + // TODO: Do we need to maintain the error state? What's triggering 'finish' after 'error'? + var errored = false + + stream.on('error', err => { + errored = true + return cb(err) + }) + stream.on('finish', () => { + if (!errored) return cb(null) + }) stream.end(buf) } @@ -497,12 +519,13 @@ class Hyperdrive extends EventEmitter { }) } - _createDirectoryStat (name, opts, cb) { + _createStat (name, opts, cb) { + const statConstructor = (opts && opts.directory) ? Stat.directory : Stat.file this._db.get(name, (err, node, trie) => { if (err) return cb(err) this._getContent(trie, (err, contentState) => { if (err) return cb(err) - const st = Stat.directory({ + const st = statConstructor({ ...opts, offset: contentState.feed.length, byteOffset: contentState.feed.byteLength @@ -519,8 +542,9 @@ class Hyperdrive extends EventEmitter { if (!cb) cb = noop name = unixify(name) + opts.directory = true - this._createDirectoryStat(name, opts, (err, st) => { + this._createStat(name, opts, (err, st) => { if (err) return cb(err) this._putStat(name, st, { condition: ifNotExists @@ -694,21 +718,51 @@ class Hyperdrive extends EventEmitter { mount (path, key, opts, cb) { if (typeof opts === 'function') return this.mount(path, key, null, opts) + const self = this + path = unixify(path) opts = opts || {} opts.mount = { key, version: opts.version, - hash: opts.hash + hash: opts.hash, + hypercore: !!opts.hypercore } - this._createDirectoryStat(path, opts, (err, st) => { - if (err) return cb(err) - this._db.mount(path, key, { ...opts, value: messages.Stat.encode(st) }, err => { + opts.directory = !opts.hypercore + + if (opts.hypercore) { + const core = this._corestore.get({ + key, + ...opts, + sparse: this.sparse + }) + core.ready(err => { if (err) return cb(err) - return this._db.loadMount(path, cb) + opts.size = core.byteLength + opts.blocks = core.length + return mountCore() }) - }) + } else { + return process.nextTick(mountTrie, null) + } + + function mountCore () { + self._createStat(path, opts, (err, st) => { + if (err) return cb(err) + return self._db.put(path, messages.Stat.encode(st), cb) + }) + } + + function mountTrie () { + self._createStat(path, opts, (err, st) => { + if (err) return cb(err) + self._db.mount(path, key, { ...opts, value: messages.Stat.encode(st) }, err => { + if (err) return cb(err) + return self._db.loadMount(path, cb) + }) + }) + } } symlink (target, linkName, cb) { diff --git a/lib/fd.js b/lib/fd.js index cc38330c..8d00afe0 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -2,7 +2,7 @@ const byteStream = require('byte-stream') const through = require('through2') const pumpify = require('pumpify') -const messages = require('./messages') +const { messages } = require('hyperdrive-schema') const errors = require('./errors') const { linux: linuxConstants, parse } = require('filesystem-constants') diff --git a/lib/storage.js b/lib/storage.js index de44fba3..6573b0fa 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -12,5 +12,5 @@ module.exports = function defaultCorestore (storage, opts) { } function isCorestore (storage) { - return storage.get && storage.replicate && storage.close + return !!storage.get && !!storage.replicate && !!storage.close } diff --git a/package.json b/package.json index 62f5aeae..e84db50f 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ }, "devDependencies": { "fuzzbuzz": "^2.0.0", + "memdb": "^1.3.1", "random-access-memory": "^3.1.1", "tape": "^4.10.0", "temporary-directory": "^1.0.2" diff --git a/schema.proto b/schema.proto index 78ff0fe0..824fc811 100644 --- a/schema.proto +++ b/schema.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message Index { required string type = 1; optional bytes content = 2; @@ -7,6 +9,7 @@ message Mount { required bytes key = 1; optional uint64 version = 2; optional bytes hash = 3; + optional bool hypercore = 4; } message Stat { diff --git a/test/helpers/create.js b/test/helpers/create.js index 65dbf151..0dc95893 100644 --- a/test/helpers/create.js +++ b/test/helpers/create.js @@ -2,5 +2,9 @@ var ram = require('random-access-memory') var hyperdrive = require('../../') module.exports = function (key, opts) { - return hyperdrive(ram, key, opts) + if (key && !(key instanceof Buffer)) { + opts = key + key = null + } + return hyperdrive((opts && opts.corestore) || ram, key, opts) } diff --git a/test/mount.js b/test/mount.js index 2dd970c0..b49c3a14 100644 --- a/test/mount.js +++ b/test/mount.js @@ -1,20 +1,24 @@ var test = require('tape') +const ram = require('random-access-memory') +const memdb = require('memdb') +const corestore = require('random-access-corestore') +const Megastore = require('megastore') var create = require('./helpers/create') test('basic read/write to/from a mount', t => { - const archive1 = create() - const archive2 = create() + const drive1 = create() + const drive2 = create() - const s1 = archive1.replicate({ live: true, encrypt: false }) - s1.pipe(archive2.replicate({ live: true, encrypt: false })).pipe(s1) + const s1 = drive1.replicate({ live: true, encrypt: false }) + s1.pipe(drive2.replicate({ live: true, encrypt: false })).pipe(s1) - archive2.ready(err => { + drive2.ready(err => { t.error(err, 'no error') - archive2.writeFile('b', 'hello', err => { + drive2.writeFile('b', 'hello', err => { t.error(err, 'no error') - archive1.mount('a', archive2.key, err => { + drive1.mount('a', drive2.key, err => { t.error(err, 'no error') - archive1.readFile('a/b', (err, contents) => { + drive1.readFile('a/b', (err, contents) => { t.error(err, 'no error') t.same(contents, Buffer.from('hello')) t.end() @@ -25,21 +29,21 @@ test('basic read/write to/from a mount', t => { }) test('readdir returns mounts', t => { - const archive1 = create() - const archive2 = create() + const drive1 = create() + const drive2 = create() - const s1 = archive1.replicate({ live: true, encrypt: false }) - s1.pipe(archive2.replicate({ live: true, encrypt: false })).pipe(s1) + const s1 = drive1.replicate({ live: true, encrypt: false }) + s1.pipe(drive2.replicate({ live: true, encrypt: false })).pipe(s1) - archive2.ready(err => { + drive2.ready(err => { t.error(err, 'no error') - archive1.mkdir('b', err => { + drive1.mkdir('b', err => { t.error(err, 'no error') - archive1.mkdir('b/a', err => { + drive1.mkdir('b/a', err => { t.error(err, 'no error') - archive1.mount('a', archive2.key, err => { + drive1.mount('a', drive2.key, err => { t.error(err, 'no error') - archive1.readdir('/', (err, dirs) => { + drive1.readdir('/', (err, dirs) => { t.error(err, 'no error') t.same(dirs, ['b', 'a']) t.end() @@ -51,22 +55,22 @@ test('readdir returns mounts', t => { }) test('cross-mount watch', t => { - const archive1 = create() - const archive2 = create() + const drive1 = create() + const drive2 = create() - const s1 = archive1.replicate({ live: true, encrypt: false }) - s1.pipe(archive2.replicate({ live: true, encrypt: false })).pipe(s1) + const s1 = drive1.replicate({ live: true, encrypt: false }) + s1.pipe(drive2.replicate({ live: true, encrypt: false })).pipe(s1) var watchEvents = 0 - archive2.ready(err => { + drive2.ready(err => { t.error(err, 'no error') - archive1.mount('a', archive2.key, err => { + drive1.mount('a', drive2.key, err => { t.error(err, 'no error') - archive1.watch('/', () => { + drive1.watch('/', () => { if (++watchEvents === 1) t.end() }) - archive2.writeFile('a', 'hello', err => { + drive2.writeFile('a', 'hello', err => { t.error(err, 'no error') }) }) @@ -74,26 +78,26 @@ test('cross-mount watch', t => { }) test('cross-mount symlink', t => { - const archive1 = create() - const archive2 = create() + const drive1 = create() + const drive2 = create() - const s1 = archive1.replicate({ live: true, encrypt: false }) - s1.pipe(archive2.replicate({ live: true, encrypt: false })).pipe(s1) + const s1 = drive1.replicate({ live: true, encrypt: false }) + s1.pipe(drive2.replicate({ live: true, encrypt: false })).pipe(s1) - archive2.ready(err => { + drive2.ready(err => { t.error(err, 'no error') - archive1.mount('a', archive2.key, err => { + drive1.mount('a', drive2.key, err => { t.error(err, 'no error') onmount() }) }) function onmount () { - archive2.writeFile('b', 'hello world', err => { + drive2.writeFile('b', 'hello world', err => { t.error(err, 'no error') - archive1.symlink('a/b', 'c', err => { + drive1.symlink('a/b', 'c', err => { t.error(err, 'no error') - archive1.readFile('c', (err, contents) => { + drive1.readFile('c', (err, contents) => { t.error(err, 'no error') t.same(contents, Buffer.from('hello world')) t.end() @@ -103,5 +107,83 @@ test('cross-mount symlink', t => { } }) +test('independent corestores do not share write capabilities', t => { + const drive1 = create() + const drive2 = create() + + const s1 = drive1.replicate({ live: true, encrypt: false }) + s1.pipe(drive2.replicate({ live: true, encrypt: false })).pipe(s1) + + drive2.ready(err => { + t.error(err, 'no error') + drive1.mount('a', drive2.key, err => { + t.error(err, 'no error') + drive1.writeFile('a/b', 'hello', err => { + t.ok(err) + drive1.readFile('a/b', (err, contents) => { + t.ok(err) + t.end() + }) + }) + }) + }) +}) + +test('shared corestores will share write capabilities', async t => { + const megastore = new Megastore(ram, memdb(), false) + await megastore.ready() + + const cs1 = megastore.get('cs1') + const cs2 = megastore.get('cs2') + + const drive1 = create({ corestore: cs1 }) + const drive2 = create({ corestore: cs2 }) + + drive2.ready(err => { + t.error(err, 'no error') + drive1.mount('a', drive2.key, err => { + t.error(err, 'no error') + drive1.writeFile('a/b', 'hello', err => { + t.error(err, 'no error') + drive1.readFile('a/b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + t.end() + }) + }) + }) + }) +}) + +test.only('can mount hypercores', async t => { + const store = corestore(ram) + const drive = create({ corestore: store }) + var core + + drive.ready(err => { + t.error(err, 'no error') + core = store.get() + core.ready(err => { + t.error(err, 'no error') + core.append('hello', err => { + t.error(err, 'no error') + return onappend() + }) + }) + }) + + function onappend () { + drive.mount('/a', core.key, { hypercore: true }, err => { + t.error(err, 'no error') + drive.readFile('/a', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + t.end() + }) + }) + } +}) + + test('versioned mount') test('watch will unwatch on umount') From 44b58756f3fc6039b5154cdfb68bf0bdaf27dfe1 Mon Sep 17 00:00:00 2001 From: "Franz K.H" Date: Wed, 12 Jun 2019 14:10:15 +0200 Subject: [PATCH 070/108] Allow to add metadata to Stat entries (#234) * Add support for metadata on stat messages. * Add documentation for metadata option. --- README.md | 3 + lib/messages.js | 205 +++++++++++++++++++++++++++++++++++++++++++++++- lib/stat.js | 1 + schema.proto | 1 + test/stat.js | 18 +++++ 5 files changed, 226 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 793bd0ae..c41d8c00 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,7 @@ If `cached` is set to `true`, this function returns results only if they have al Write a file as a stream. Similar to fs.createWriteStream. If `options.cached` is set to `true`, this function returns results only if they have already been downloaded. +`options.metadata` is optionally an object with string keys and buffer objects to set metadata on the file entry. #### `archive.writeFile(name, buffer, [options], [callback])` @@ -236,6 +237,8 @@ Stat { linkname: undefined } ``` +The stat may include a metadata object (string keys, buffer values) with metadata that was passed into `writeFile` or `createWriteStream`. + The output object includes methods similar to fs.stat: ``` js diff --git a/lib/messages.js b/lib/messages.js index 9fc819d2..b65de5ec 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -17,6 +17,13 @@ var Index = exports.Index = { decode: null } +var Metadata = exports.Metadata = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + var Stat = exports.Stat = { buffer: true, encodingLength: null, @@ -24,8 +31,17 @@ var Stat = exports.Stat = { decode: null } +var Map_string_bytes = exports.Map_string_bytes = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + defineIndex() +defineMetadata() defineStat() +defineMap_string_bytes() function defineIndex () { var enc = [ @@ -102,9 +118,85 @@ function defineIndex () { } } +function defineMetadata () { + var enc = [ + encodings.string, + encodings.bytes + ] + + Metadata.encodingLength = encodingLength + Metadata.encode = encode + Metadata.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.type)) throw new Error("type is required") + var len = enc[0].encodingLength(obj.type) + length += 1 + len + if (!defined(obj.content)) throw new Error("content is required") + var len = enc[1].encodingLength(obj.content) + length += 1 + len + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.type)) throw new Error("type is required") + buf[offset++] = 10 + enc[0].encode(obj.type, buf, offset) + offset += enc[0].encode.bytes + if (!defined(obj.content)) throw new Error("content is required") + buf[offset++] = 18 + enc[1].encode(obj.content, buf, offset) + offset += enc[1].encode.bytes + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + type: "", + content: null + } + var found0 = false + var found1 = false + while (true) { + if (end <= offset) { + if (!found0 || !found1) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.type = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + found0 = true + break + case 2: + obj.content = enc[1].decode(buf, offset) + offset += enc[1].decode.bytes + found1 = true + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + function defineStat () { var enc = [ - encodings.varint + encodings.varint, + Map_string_bytes ] Stat.encodingLength = encodingLength @@ -148,6 +240,18 @@ function defineStat () { var len = enc[0].encodingLength(obj.ctime) length += 1 + len } + if (defined(obj.metadata)) { + var tmp = Object.keys(obj.metadata) + for (var i = 0; i < tmp.length; i++) { + tmp[i] = {key: tmp[i], value: obj.metadata[tmp[i]]} + } + for (var i = 0; i < tmp.length; i++) { + if (!defined(tmp[i])) continue + var len = enc[1].encodingLength(tmp[i]) + length += varint.encodingLength(len) + length += 1 + len + } + } return length } @@ -199,6 +303,20 @@ function defineStat () { enc[0].encode(obj.ctime, buf, offset) offset += enc[0].encode.bytes } + if (defined(obj.metadata)) { + var tmp = Object.keys(obj.metadata) + for (var i = 0; i < tmp.length; i++) { + tmp[i] = {key: tmp[i], value: obj.metadata[tmp[i]]} + } + for (var i = 0; i < tmp.length; i++) { + if (!defined(tmp[i])) continue + buf[offset++] = 82 + varint.encode(enc[1].encodingLength(tmp[i]), buf, offset) + offset += varint.encode.bytes + enc[1].encode(tmp[i], buf, offset) + offset += enc[1].encode.bytes + } + } encode.bytes = offset - oldOffset return buf } @@ -217,7 +335,8 @@ function defineStat () { offset: 0, byteOffset: 0, mtime: 0, - ctime: 0 + ctime: 0, + metadata: {} } var found0 = false while (true) { @@ -267,6 +386,88 @@ function defineStat () { obj.ctime = enc[0].decode(buf, offset) offset += enc[0].decode.bytes break + case 10: + var len = varint.decode(buf, offset) + offset += varint.decode.bytes + var tmp = enc[1].decode(buf, offset, offset + len) + obj.metadata[tmp.key] = tmp.value + offset += enc[1].decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defineMap_string_bytes () { + var enc = [ + encodings.string, + encodings.bytes + ] + + Map_string_bytes.encodingLength = encodingLength + Map_string_bytes.encode = encode + Map_string_bytes.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.key)) throw new Error("key is required") + var len = enc[0].encodingLength(obj.key) + length += 1 + len + if (defined(obj.value)) { + var len = enc[1].encodingLength(obj.value) + length += 1 + len + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.key)) throw new Error("key is required") + buf[offset++] = 10 + enc[0].encode(obj.key, buf, offset) + offset += enc[0].encode.bytes + if (defined(obj.value)) { + buf[offset++] = 18 + enc[1].encode(obj.value, buf, offset) + offset += enc[1].encode.bytes + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + key: "", + value: null + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.key = enc[0].decode(buf, offset) + offset += enc[0].decode.bytes + found0 = true + break + case 2: + obj.value = enc[1].decode(buf, offset) + offset += enc[1].decode.bytes + break default: offset = skip(prefix & 7, buf, offset) } diff --git a/lib/stat.js b/lib/stat.js index 79ee1588..a9734e8a 100644 --- a/lib/stat.js +++ b/lib/stat.js @@ -21,6 +21,7 @@ class Stat { this.mtime = data && data.mtime ? getTime(data.mtime) : Date.now() this.ctime = data && data.ctime ? getTime(data.ctime) : Date.now() this.linkname = (data && data.linkname) || null + this.metadata = (data && data.metadata) || null } _check (mask) { diff --git a/schema.proto b/schema.proto index 6a89ff2b..71c6dcfa 100644 --- a/schema.proto +++ b/schema.proto @@ -13,4 +13,5 @@ message Stat { optional uint64 byteOffset = 7; optional uint64 mtime = 8; optional uint64 ctime = 9; + map metadata = 10; } diff --git a/test/stat.js b/test/stat.js index 2afe3369..9cc87305 100644 --- a/test/stat.js +++ b/test/stat.js @@ -35,3 +35,21 @@ tape('stat dir', function (t) { }) }) }) + +tape('metadata', function (t) { + var archive = create() + + var attributes = { hello: 'world' } + var metadata = { + attributes: Buffer.from(JSON.stringify(attributes)) + } + + archive.writeFile('/foo', 'bar', { metadata }, function (err) { + t.error(err, 'no error') + archive.stat('/foo', function (err, st) { + t.error(err, 'no error') + t.deepEqual(JSON.parse(metadata.attributes.toString()), { hello: 'world' }) + t.end() + }) + }) +}) From b2c08a338722d1b0f2d31490f0cae5654d1388cb Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 14 Jun 2019 11:08:14 +0200 Subject: [PATCH 071/108] More changes for mounts --- index.js | 58 ++++++++-------- lib/fd.js | 9 ++- test/basic.js | 60 ++++++++--------- test/mount.js | 178 +++++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 236 insertions(+), 69 deletions(-) diff --git a/index.js b/index.js index eca9f2e6..18af523d 100644 --- a/index.js +++ b/index.js @@ -18,7 +18,7 @@ const createFileDescriptor = require('./lib/fd') const Stat = require('./lib/stat') const errors = require('./lib/errors') const defaultCorestore = require('./lib/storage') -const { messages } = require('hyperdrive-schema') +const { messages } = require('hyperdrive-schemas') const { contentKeyPair, contentOptions, ContentState } = require('./lib/content') // 20 is arbitrary, just to make the fds > stdio etc @@ -41,19 +41,21 @@ class Hyperdrive extends EventEmitter { this.key = null this.discoveryKey = null this.live = true - this.sparse = !!opts.sparse - this.sparseMetadata = !!opts.sparseMetadata + this.sparse = opts.sparse !== false + this.sparseMetadata = opts.sparseMetadata !== false - this._corestore = defaultCorestore(storage, opts) + // TODO: Add support for mixed-sparsity. + this._corestore = defaultCorestore(storage, { + sparse: this.sparse, + valueEncoding: 'binary' + }) this.metadata = this._corestore.default({ key, secretKey: opts.secretKey, - sparse: this.sparseMetadata, - createIfMissing: opts.createIfMissing, - storageCacheSize: opts.metadataStorageCacheSize, - valueEncoding: 'binary' }) - this._db = opts._db || new MountableHypertrie(this._corestore, key, { feed: this.metadata }) + this._db = opts._db || new MountableHypertrie(this._corestore, key, { + feed: this.metadata + }) this._contentStates = new Map() if (opts.content) this._contentStates.set(this._db, new ContentState(opts.content)) @@ -445,7 +447,7 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return cb(err) - this.lstat(name, { file: true }, (err, stat) => { + this.lstat(name, { file: true, trie: true }, (err, stat) => { if (err && err.errno !== 2) return cb(err) if (stat) return cb(null, stat) const st = Stat.file(opts) @@ -495,27 +497,24 @@ class Hyperdrive extends EventEmitter { truncate (name, size, cb) { name = unixify(name) - this.contentReady(err => { - if (err) return cb(err) - this.lstat(name, { file: true }, (err, st) => { - if (err && err.errno !== 2) return cb(err) - if (!st) return this.create(name, cb) - if (size === st.size) return cb(null) - if (size < st.size) { - const readStream = this.createReadStream(name, { length: size }) - const writeStream = this.createWriteStream(name) - return pump(readStream, writeStream, cb) - } else { - this.open(name, 'a', (err, fd) => { + this.lstat(name, { file: true, trie: true }, (err, st) => { + if (err && err.errno !== 2) return cb(err) + if (!st) return this.create(name, cb) + if (size === st.size) return cb(null) + if (size < st.size) { + const readStream = this.createReadStream(name, { length: size }) + const writeStream = this.createWriteStream(name) + return pump(readStream, writeStream, cb) + } else { + this.open(name, 'a', (err, fd) => { + if (err) return cb(err) + const length = size - st.size + this.write(fd, Buffer.alloc(length), 0, length, st.size, err => { if (err) return cb(err) - const length = size - st.size - this.write(fd, Buffer.alloc(length), 0, length, st.size, err => { - if (err) return cb(err) - this.close(fd, cb) - }) + this.close(fd, cb) }) - } - }) + }) + } }) } @@ -718,6 +717,7 @@ class Hyperdrive extends EventEmitter { mount (path, key, opts, cb) { if (typeof opts === 'function') return this.mount(path, key, null, opts) + console.error('MOUNTING KEY:', key, 'AT PATH:', path, 'WITH OPTS:', opts) const self = this path = unixify(path) diff --git a/lib/fd.js b/lib/fd.js index 8d00afe0..4899ee85 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -2,7 +2,7 @@ const byteStream = require('byte-stream') const through = require('through2') const pumpify = require('pumpify') -const { messages } = require('hyperdrive-schema') +const { messages } = require('hyperdrive-schemas') const errors = require('./errors') const { linux: linuxConstants, parse } = require('filesystem-constants') @@ -48,6 +48,11 @@ class FileDescriptor { } read (buffer, offset, len, pos, cb) { + const oldcb = cb + cb = (err, results) => { + console.error('FILE DESCRIPTOR READ ERR:', err, 'RESULTS:', results) + return process.nextTick(oldcb, err, results) + } if (!this.readable) return cb(new errors.BadFileDescriptor('File descriptor not open for reading.')) if (this.position !== null && this.position === pos) this._read(buffer, offset, len, cb) else this._seekAndRead(buffer, offset, len, pos, cb) @@ -242,6 +247,8 @@ module.exports = function create (drive, name, flags, cb) { if (st && !canExist) return cb(new errors.PathAlreadyExists(name)) if (!st && (!writable || !creating)) return cb(new errors.FileNotFound(name)) + console.error('IN HYPERDRIVE, FD STAT:', st) + drive._getContent(trie, (err, contentState) => { if (err) return cb(err) const fd = new FileDescriptor(drive, name, st, contentState, readable, writable, appending, creating) diff --git a/test/basic.js b/test/basic.js index d167b762..b150d710 100644 --- a/test/basic.js +++ b/test/basic.js @@ -3,11 +3,11 @@ var sodium = require('sodium-universal') var create = require('./helpers/create') tape('write and read', function (t) { - var archive = create() + var drive = create() - archive.writeFile('/hello.txt', 'world', function (err) { + drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - archive.readFile('/hello.txt', function (err, buf) { + drive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') t.same(buf, Buffer.from('world')) t.end() @@ -16,11 +16,11 @@ tape('write and read', function (t) { }) tape('write and read, with encoding', function (t) { - var archive = create() + var drive = create() - archive.writeFile('/hello.txt', 'world', { encoding: 'utf8' }, function (err) { + drive.writeFile('/hello.txt', 'world', { encoding: 'utf8' }, function (err) { t.error(err, 'no error') - archive.readFile('/hello.txt', { encoding: 'utf8' }, function (err, str) { + drive.readFile('/hello.txt', { encoding: 'utf8' }, function (err, str) { t.error(err, 'no error') t.same(str, 'world') t.end() @@ -31,19 +31,19 @@ tape('write and read, with encoding', function (t) { tape('write and read (2 parallel)', function (t) { t.plan(6) - var archive = create() + var drive = create() - archive.writeFile('/hello.txt', 'world', function (err) { + drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - archive.readFile('/hello.txt', function (err, buf) { + drive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') t.same(buf, Buffer.from('world')) }) }) - archive.writeFile('/world.txt', 'hello', function (err) { + drive.writeFile('/world.txt', 'hello', function (err) { t.error(err, 'no error') - archive.readFile('/world.txt', function (err, buf) { + drive.readFile('/world.txt', function (err, buf) { t.error(err, 'no error') t.same(buf, Buffer.from('hello')) }) @@ -53,15 +53,15 @@ tape('write and read (2 parallel)', function (t) { tape('write and read (sparse)', function (t) { t.plan(2) - var archive = create() - archive.on('ready', function () { - var clone = create(archive.key, {sparse: true}) + var drive = create() + drive.on('ready', function () { + var clone = create(drive.key, {sparse: true}) - archive.writeFile('/hello.txt', 'world', function (err) { + drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') var s1 = clone.replicate({ live: true }) - var s2 = archive.replicate({ live: true }) - // stream.pipe(archive.replicate()).pipe(stream) + var s2 = drive.replicate({ live: true }) + // stream.pipe(drive.replicate()).pipe(stream) s1.pipe(s2).pipe(s1) setTimeout(() => { var readStream = clone.createReadStream('/hello.txt') @@ -74,11 +74,11 @@ tape('write and read (sparse)', function (t) { }) tape('root is always there', function (t) { - var archive = create() + var drive = create() - archive.access('/', function (err) { + drive.access('/', function (err) { t.error(err, 'no error') - archive.readdir('/', function (err, list) { + drive.readdir('/', function (err, list) { t.error(err, 'no error') t.same(list, []) t.end() @@ -92,16 +92,16 @@ tape('provide keypair', function (t) { sodium.crypto_sign_keypair(publicKey, secretKey) - var archive = create(publicKey, {secretKey: secretKey}) + var drive = create(publicKey, {secretKey: secretKey}) - archive.on('ready', function () { - t.ok(archive.writable) - t.ok(archive.metadata.writable) - t.ok(publicKey.equals(archive.key)) + drive.on('ready', function () { + t.ok(drive.writable) + t.ok(drive.metadata.writable) + t.ok(publicKey.equals(drive.key)) - archive.writeFile('/hello.txt', 'world', function (err) { + drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - archive.readFile('/hello.txt', function (err, buf) { + drive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') t.same(buf, Buffer.from('world')) t.end() @@ -111,15 +111,15 @@ tape('provide keypair', function (t) { }) tape('write and read, no cache', function (t) { - var archive = create({ + var drive = create({ metadataStorageCacheSize: 0, contentStorageCacheSize: 0, treeCacheSize: 0 }) - archive.writeFile('/hello.txt', 'world', function (err) { + drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - archive.readFile('/hello.txt', function (err, buf) { + drive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') t.same(buf, Buffer.from('world')) t.end() diff --git a/test/mount.js b/test/mount.js index b49c3a14..d442455f 100644 --- a/test/mount.js +++ b/test/mount.js @@ -5,7 +5,7 @@ const corestore = require('random-access-corestore') const Megastore = require('megastore') var create = require('./helpers/create') -test('basic read/write to/from a mount', t => { +test.only('basic read/write to/from a mount', t => { const drive1 = create() const drive2 = create() @@ -18,14 +18,118 @@ test('basic read/write to/from a mount', t => { t.error(err, 'no error') drive1.mount('a', drive2.key, err => { t.error(err, 'no error') - drive1.readFile('a/b', (err, contents) => { + setTimeout(() => { + drive1.readFile('a/b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + t.end() + }) + }, 5000) + }) + }) + }) +}) + +test('multiple flat mounts', t => { + const drive1 = create() + const drive2 = create() + const drive3 = create() + + var key1, key2 + + replicateAll([drive1, drive2, drive3]) + + drive3.ready(err => { + drive2.ready(err => { + key1 = drive2.key + key2 = drive3.key + onready() + }) + }) + + function onready () { + drive2.writeFile('a', 'hello', err => { + t.error(err, 'no error') + drive3.writeFile('b', 'world', err => { + t.error(err, 'no error') + onwrite() + }) + }) + } + + function onwrite () { + drive1.mount('a', key1, err => { + t.error(err, 'no error') + drive1.mount('b', key2, err => { + t.error(err, 'no error') + onmount() + }) + }) + } + + function onmount () { + drive1.readFile('a/a', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + drive1.readFile('b/b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('world')) + t.end() + }) + }) + } +}) + +test('recursive mounts', async t => { + var key1, key2 + const drive1 = create() + const drive2 = create() + const drive3 = create() + + replicateAll([drive1, drive2, drive3]) + + drive3.ready(err => { + drive2.ready(err => { + key1 = drive2.key + key2 = drive3.key + onready() + }) + }) + + function onready () { + drive2.writeFile('a', 'hello', err => { + t.error(err, 'no error') + drive3.writeFile('b', 'world', err => { + t.error(err, 'no error') + console.log('DRIVE 3 KEY:', drive3.key) + onwrite() + }) + }) + } + + function onwrite () { + drive1.mount('a', key1, err => { + t.error(err, 'no error') + drive2.mount('b', key2, err => { + t.error(err, 'no error') + onmount() + }) + }) + } + + function onmount () { + drive1.readFile('a/a', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + setTimeout(() => { + drive1.readFile('a/b/b', (err, contents) => { t.error(err, 'no error') - t.same(contents, Buffer.from('hello')) + t.same(contents, Buffer.from('world')) t.end() }) - }) + }, 2000) }) - }) + } }) test('readdir returns mounts', t => { @@ -148,21 +252,24 @@ test('shared corestores will share write capabilities', async t => { drive1.readFile('a/b', (err, contents) => { t.error(err, 'no error') t.same(contents, Buffer.from('hello')) - t.end() + drive2.readFile('b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + t.end() + }) }) }) }) }) }) -test.only('can mount hypercores', async t => { +test('can mount hypercores', async t => { const store = corestore(ram) const drive = create({ corestore: store }) - var core + var core = store.get() drive.ready(err => { t.error(err, 'no error') - core = store.get() core.ready(err => { t.error(err, 'no error') core.append('hello', err => { @@ -184,6 +291,59 @@ test.only('can mount hypercores', async t => { } }) +test('truncate within mount (with shared write capabilities)', async t => { + const megastore = new Megastore(ram, memdb(), false) + await megastore.ready() + + const cs1 = megastore.get('cs1') + const cs2 = megastore.get('cs2') + + const drive1 = create({ corestore: cs1 }) + const drive2 = create({ corestore: cs2 }) + + drive2.ready(err => { + t.error(err, 'no error') + drive1.mount('a', drive2.key, err => { + t.error(err, 'no error') + drive1.writeFile('a/b', 'hello', err => { + t.error(err, 'no error') + drive1.truncate('a/b', 1, err => { + t.error(err, 'no error') + drive1.readFile('a/b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('h')) + drive2.readFile('b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('h')) + t.end() + }) + }) + }) + }) + }) + }) +}) + test('versioned mount') test('watch will unwatch on umount') + +function replicateAll (drives) { + const streams = [] + for (let i = 0; i < drives.length; i++) { + for (let j = 0; j < drives.length; j++) { + const source = drives[i] + const dest = drives[j] + if (i === j) continue + + const s1 = source.replicate({ live: true, encrypt: false}) + const s2 = dest.replicate({ live: true, encrypt: false }) + streams.push([s1, s2]) + + s1.on('data', d => console.log(`${i + 1} STREAM DATA:`, d)) + s2.on('data', d => console.log(`${j + 1} STREAM DATA:`, d)) + s1.pipe(s2).pipe(s1) + } + } + return streams +} From ab2a961c7d1964640671fe6a0755acbc4709b671 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 14 Jun 2019 15:11:51 +0200 Subject: [PATCH 072/108] Enabled more tests + bug fixes --- index.js | 34 ++++++++++++-------------- lib/content.js | 7 +++--- lib/fd.js | 7 ------ test/basic.js | 12 +++++----- test/checkout.js | 2 +- test/index.js | 2 +- test/mount.js | 48 ++++++++++++++++++------------------- test/storage.js | 62 ++++++++++++++++++++++++------------------------ 8 files changed, 81 insertions(+), 93 deletions(-) diff --git a/index.js b/index.js index 18af523d..13d27a2a 100644 --- a/index.js +++ b/index.js @@ -1,4 +1,4 @@ -const path = require('path') +const path = require('path').posix const { EventEmitter } = require('events') const collect = require('stream-collector') @@ -42,9 +42,9 @@ class Hyperdrive extends EventEmitter { this.discoveryKey = null this.live = true this.sparse = opts.sparse !== false - this.sparseMetadata = opts.sparseMetadata !== false - // TODO: Add support for mixed-sparsity. + this.sparseMetadata = this.sparse || opts.sparseMetadata !== false + this._corestore = defaultCorestore(storage, { sparse: this.sparse, valueEncoding: 'binary' @@ -54,11 +54,12 @@ class Hyperdrive extends EventEmitter { secretKey: opts.secretKey, }) this._db = opts._db || new MountableHypertrie(this._corestore, key, { - feed: this.metadata + feed: this.metadata, + sparse: this.sparse }) this._contentStates = new Map() - if (opts.content) this._contentStates.set(this._db, new ContentState(opts.content)) + if (opts._content) this._contentStates.set(this._db, new ContentState(opts._content)) this._fds = [] this._writingFds = new Map() @@ -98,15 +99,11 @@ class Hyperdrive extends EventEmitter { return self.metadata.ready(err => { if (err) return cb(err) - if (self.sparseMetadata) { - self.metadata.update(function loop () { - self.metadata.update(loop) - }) - } - const rootContentKeyPair = self.metadata.secretKey ? contentKeyPair(self.metadata.secretKey) : {} /** + * TODO: Update comment to reflect mounts. + * * If a db is provided as input, ensure that a contentFeed is also provided, then return (this is a checkout). * If the metadata feed is writable: * If the metadata feed has length 0, then the db should be initialized with the content feed key as metadata. @@ -115,7 +112,7 @@ class Hyperdrive extends EventEmitter { * Initialize the db without metadata and load the content feed key from the header. */ if (self.opts._db) { - if (!self.contentStates.get(self.opts._db.key)) return cb(new Error('Must provide a db and a content feed')) + if (!self._contentStates.get(self.opts._db)) return cb(new Error('Must provide a db and a content feed')) return done(null) } else if (self.metadata.writable && !self.metadata.length) { initialize(rootContentKeyPair) @@ -187,7 +184,7 @@ class Hyperdrive extends EventEmitter { const feed = self._corestore.get(contentOpts) feed.ready(err => { if (err) return cb(err) - const state = new ContentState(feed, mutexify()) + const state = new ContentState(feed) self._contentStates.set(db, state) feed.on('error', err => self.emit('error', err)) return cb(null, state) @@ -674,17 +671,17 @@ class Hyperdrive extends EventEmitter { } replicate (opts) { - const stream = this._corestore.replicate(opts) - stream.on('error', err => console.error('REPLICATION ERROR:', err)) - return stream + return this._corestore.replicate(opts) } checkout (version, opts) { + const db = this._db.checkout(version) opts = { ...opts, - _db: this._db.checkout(version) + _db: db, + _content: this._contentStates.get(this._db) } - return new Hyperdrive(this.storage, this.key, opts) + return new Hyperdrive(this._corestore, this.key, opts) } _closeFile (fd, cb) { @@ -717,7 +714,6 @@ class Hyperdrive extends EventEmitter { mount (path, key, opts, cb) { if (typeof opts === 'function') return this.mount(path, key, null, opts) - console.error('MOUNTING KEY:', key, 'AT PATH:', path, 'WITH OPTS:', opts) const self = this path = unixify(path) diff --git a/lib/content.js b/lib/content.js index 7f091c20..32cf5434 100644 --- a/lib/content.js +++ b/lib/content.js @@ -1,3 +1,4 @@ +const mutexify = require('mutexify') const sodium = require('sodium-universal') function contentKeyPair (secretKey) { @@ -26,9 +27,9 @@ function contentOptions (self, secretKey) { } class ContentState { - constructor (feed, lock) { - this.feed = feed - this._lock = lock + constructor (feed) { + this.feed = (feed instanceof ContentState) ? feed.feed : feed + this._lock = mutexify() } lock (cb) { return this._lock(cb) diff --git a/lib/fd.js b/lib/fd.js index 4899ee85..68ad19a5 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -48,11 +48,6 @@ class FileDescriptor { } read (buffer, offset, len, pos, cb) { - const oldcb = cb - cb = (err, results) => { - console.error('FILE DESCRIPTOR READ ERR:', err, 'RESULTS:', results) - return process.nextTick(oldcb, err, results) - } if (!this.readable) return cb(new errors.BadFileDescriptor('File descriptor not open for reading.')) if (this.position !== null && this.position === pos) this._read(buffer, offset, len, cb) else this._seekAndRead(buffer, offset, len, pos, cb) @@ -247,8 +242,6 @@ module.exports = function create (drive, name, flags, cb) { if (st && !canExist) return cb(new errors.PathAlreadyExists(name)) if (!st && (!writable || !creating)) return cb(new errors.FileNotFound(name)) - console.error('IN HYPERDRIVE, FD STAT:', st) - drive._getContent(trie, (err, contentState) => { if (err) return cb(err) const fd = new FileDescriptor(drive, name, st, contentState, readable, writable, appending, creating) diff --git a/test/basic.js b/test/basic.js index b150d710..878d2de3 100644 --- a/test/basic.js +++ b/test/basic.js @@ -57,18 +57,18 @@ tape('write and read (sparse)', function (t) { drive.on('ready', function () { var clone = create(drive.key, {sparse: true}) + var s1 = clone.replicate({ live: true, encrypt: false }) + var s2 = drive.replicate({ live: true, encrypt: false}) + s1.pipe(s2).pipe(s1) + drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - var s1 = clone.replicate({ live: true }) - var s2 = drive.replicate({ live: true }) - // stream.pipe(drive.replicate()).pipe(stream) - s1.pipe(s2).pipe(s1) setTimeout(() => { var readStream = clone.createReadStream('/hello.txt') readStream.on('data', function (data) { t.same(data.toString(), 'world') - }) - }, 100) + }, 50) + }) }) }) }) diff --git a/test/checkout.js b/test/checkout.js index b59d80fd..b51c5574 100644 --- a/test/checkout.js +++ b/test/checkout.js @@ -2,7 +2,7 @@ var tape = require('tape') var create = require('./helpers/create') tape('simple checkout', function (t) { - const drive = create(null) + const drive = create() drive.writeFile('/hello', 'world', err => { t.error(err, 'no error') diff --git a/test/index.js b/test/index.js index 888f61c5..e1e1a3f5 100644 --- a/test/index.js +++ b/test/index.js @@ -1,5 +1,5 @@ require('./basic') -//require('./checkout') +require('./checkout') require('./creation') require('./deletion') // require('./diff') diff --git a/test/mount.js b/test/mount.js index d442455f..ec87c206 100644 --- a/test/mount.js +++ b/test/mount.js @@ -5,7 +5,7 @@ const corestore = require('random-access-corestore') const Megastore = require('megastore') var create = require('./helpers/create') -test.only('basic read/write to/from a mount', t => { +test('basic read/write to/from a mount', t => { const drive1 = create() const drive2 = create() @@ -14,17 +14,14 @@ test.only('basic read/write to/from a mount', t => { drive2.ready(err => { t.error(err, 'no error') - drive2.writeFile('b', 'hello', err => { - t.error(err, 'no error') + drive2.writeFile('b', 'hello', err => {t.error(err, 'no error') drive1.mount('a', drive2.key, err => { t.error(err, 'no error') - setTimeout(() => { - drive1.readFile('a/b', (err, contents) => { - t.error(err, 'no error') - t.same(contents, Buffer.from('hello')) - t.end() - }) - }, 5000) + drive1.readFile('a/b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + t.end() + }) }) }) }) @@ -101,7 +98,6 @@ test('recursive mounts', async t => { t.error(err, 'no error') drive3.writeFile('b', 'world', err => { t.error(err, 'no error') - console.log('DRIVE 3 KEY:', drive3.key) onwrite() }) }) @@ -121,13 +117,11 @@ test('recursive mounts', async t => { drive1.readFile('a/a', (err, contents) => { t.error(err, 'no error') t.same(contents, Buffer.from('hello')) - setTimeout(() => { - drive1.readFile('a/b/b', (err, contents) => { - t.error(err, 'no error') - t.same(contents, Buffer.from('world')) - t.end() - }) - }, 2000) + drive1.readFile('a/b/b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('world')) + t.end() + }) }) } }) @@ -211,6 +205,9 @@ test('cross-mount symlink', t => { } }) +test('dynamically resolves cross-mount symlinks') +test('symlinks cannot break the sandbox') + test('independent corestores do not share write capabilities', t => { const drive1 = create() const drive2 = create() @@ -324,26 +321,27 @@ test('truncate within mount (with shared write capabilities)', async t => { }) }) - test('versioned mount') test('watch will unwatch on umount') -function replicateAll (drives) { +function replicateAll (drives, opts) { const streams = [] + const replicated = new Set() + for (let i = 0; i < drives.length; i++) { for (let j = 0; j < drives.length; j++) { const source = drives[i] const dest = drives[j] - if (i === j) continue + if (i === j || replicated.has(j)) continue - const s1 = source.replicate({ live: true, encrypt: false}) - const s2 = dest.replicate({ live: true, encrypt: false }) + const s1 = source.replicate({ ...opts, live: true, encrypt: false}) + const s2 = dest.replicate({ ...opts, live: true, encrypt: false }) streams.push([s1, s2]) - s1.on('data', d => console.log(`${i + 1} STREAM DATA:`, d)) - s2.on('data', d => console.log(`${j + 1} STREAM DATA:`, d)) s1.pipe(s2).pipe(s1) } + replicated.add(i) } + return streams } diff --git a/test/storage.js b/test/storage.js index 80f72382..adebbac9 100644 --- a/test/storage.js +++ b/test/storage.js @@ -4,11 +4,11 @@ const create = require('./helpers/create') const hyperdrive = require('..') tape('ram storage', function (t) { - var archive = create() + var drive = create() - archive.ready(function () { - t.ok(archive.metadata.writable, 'archive metadata is writable') - t.ok(archive.contentWritable, 'archive content is writable') + drive.ready(function () { + t.ok(drive.metadata.writable, 'drive metadata is writable') + t.ok(drive.contentWritable, 'drive content is writable') t.end() }) }) @@ -16,20 +16,20 @@ tape('ram storage', function (t) { tape('dir storage with resume', function (t) { tmp(function (err, dir, cleanup) { t.ifError(err) - var archive = hyperdrive(dir) - archive.ready(function () { - t.ok(archive.metadata.writable, 'archive metadata is writable') - t.ok(archive.contentWritable, 'archive content is writable') - t.same(archive.version, 1, 'archive has version 1') - archive.close(function (err) { + var drive = hyperdrive(dir) + drive.ready(function () { + t.ok(drive.metadata.writable, 'drive metadata is writable') + t.ok(drive.contentWritable, 'drive content is writable') + t.same(drive.version, 1, 'drive has version 1') + drive.close(function (err) { t.ifError(err) - var archive2 = hyperdrive(dir) - archive2.ready(function (err) { + var drive2 = hyperdrive(dir) + drive2.ready(function (err) { t.error(err, 'no error') - t.ok(archive2.metadata.writable, 'archive2 metadata is writable') - t.ok(archive2.contentWritable, 'archive2 content is writable') - t.same(archive2.version, 1, 'archive has version 1') + t.ok(drive2.metadata.writable, 'drive2 metadata is writable') + t.ok(drive2.contentWritable, 'drive2 content is writable') + t.same(drive2.version, 1, 'drive has version 1') cleanup(function (err) { t.ifError(err) @@ -41,7 +41,7 @@ tape('dir storage with resume', function (t) { }) }) -tape('dir storage for non-writable archive', function (t) { +tape('dir storage for non-writable drive', function (t) { var src = create() src.ready(function () { tmp(function (err, dir, cleanup) { @@ -66,8 +66,8 @@ tape('dir storage for non-writable archive', function (t) { tape('dir storage without permissions emits error', function (t) { t.plan(1) - var archive = hyperdrive('/') - archive.on('error', function (err) { + var drive = hyperdrive('/') + drive.on('error', function (err) { t.ok(err, 'got error') }) }) @@ -77,14 +77,14 @@ tape('write and read (sparse)', function (t) { tmp(function (err, dir, cleanup) { t.ifError(err) - var archive = hyperdrive(dir) - archive.on('ready', function () { - var clone = create(archive.key, {sparse: true}) + var drive = hyperdrive(dir) + drive.on('ready', function () { + var clone = create(drive.key, {sparse: true}) clone.on('ready', function () { - archive.writeFile('/hello.txt', 'world', function (err) { + drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - var stream = clone.replicate({ live: true }) - stream.pipe(archive.replicate({ live: true })).pipe(stream) + var stream = clone.replicate({ live: true, encrypt: false }) + stream.pipe(drive.replicate({ live: true, encrypt: false })).pipe(stream) setTimeout(() => { var readStream = clone.createReadStream('/hello.txt') readStream.on('error', function (err) { @@ -101,15 +101,15 @@ tape('write and read (sparse)', function (t) { }) tape('sparse read/write two files', function (t) { - var archive = create() - archive.on('ready', function () { - var clone = create(archive.key, {sparse: true}) - archive.writeFile('/hello.txt', 'world', function (err) { + var drive = create() + drive.on('ready', function () { + var clone = create(drive.key, {sparse: true}) + drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - archive.writeFile('/hello2.txt', 'world', function (err) { + drive.writeFile('/hello2.txt', 'world', function (err) { t.error(err, 'no error') - var stream = clone.replicate({ live: true }) - stream.pipe(archive.replicate({ live: true })).pipe(stream) + var stream = clone.replicate({ live: true, encrypt: false }) + stream.pipe(drive.replicate({ live: true, encrypt: false })).pipe(stream) clone.metadata.update(start) }) }) From f498a0ba9febdf782a038eb8501ff6655a3f5ed5 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 15 Jun 2019 15:26:02 +0200 Subject: [PATCH 073/108] Temporary deps + update content feeds on creation --- index.js | 11 ++++++++++- package.json | 9 +++++---- test/fuzzing.js | 26 +++++++++++++++----------- test/index.js | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/index.js b/index.js index 13d27a2a..cf670b1f 100644 --- a/index.js +++ b/index.js @@ -10,7 +10,6 @@ const through = require('through2') const pumpify = require('pumpify') const pump = require('pump') -const hypercore = require('hypercore') const coreByteStream = require('hypercore-byte-stream') const MountableHypertrie = require('mountable-hypertrie') @@ -187,6 +186,12 @@ class Hyperdrive extends EventEmitter { const state = new ContentState(feed) self._contentStates.set(db, state) feed.on('error', err => self.emit('error', err)) + if (!feed.writable) { + return feed.update(1, err => { + if (err) return cb(err) + return cb(null, state) + }) + } return cb(null, state) }) } @@ -281,6 +286,7 @@ class Hyperdrive extends EventEmitter { }) function oncontent (st, contentState) { + console.log('IN ONCONTENT, content length:', contentState.feed.length, 'byteLength:', contentState.feed.byteLength) if (st.mount && st.mount.hypercore) { var byteOffset = 0 var blockOffset = 0 @@ -297,6 +303,9 @@ class Hyperdrive extends EventEmitter { } const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) + console.log('st:', st) + console.log('byteLength:', byteLength, 'blockOffset:', blockOffset, 'byteOffset:', byteOffset, 'blockLength:', blockLength) + stream.start({ feed, blockOffset, diff --git a/package.json b/package.json index e84db50f..422c1341 100644 --- a/package.json +++ b/package.json @@ -25,13 +25,13 @@ "custom-error-class": "^1.0.0", "duplexify": "^3.7.1", "filesystem-constants": "^1.0.0", - "hypercore": "^6.25.0", "hypercore-byte-stream": "^1.0.2", - "mountable-hypertrie": "^0.9.0", - "random-access-corestore": "^0.9.0", + "hyperdrive-schemas": "^0.9.0", + "mountable-hypertrie": "git+https://github.com/andrewosh/mountable-hypertrie#master", "mutexify": "^1.2.0", "pump": "^3.0.0", "pumpify": "^1.5.1", + "random-access-corestore": "git+https://github.com/andrewosh/random-access-corestore#master", "sodium-universal": "^2.0.0", "stream-collector": "^1.0.1", "through2": "^3.0.0", @@ -43,6 +43,7 @@ "memdb": "^1.3.1", "random-access-memory": "^3.1.1", "tape": "^4.10.0", - "temporary-directory": "^1.0.2" + "temporary-directory": "^1.0.2", + "megastore": "git+https://github.com/andrewosh/megastore" } } diff --git a/test/fuzzing.js b/test/fuzzing.js index 54fe9bc9..cd0f42cf 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -21,12 +21,12 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(5, this.randomStatefulFileDescriptorRead) this.add(5, this.randomStatefulFileDescriptorWrite) this.add(3, this.statFile) - this.add(3, this.statDirectory) + //this.add(3, this.statDirectory) this.add(2, this.deleteInvalidFile) this.add(2, this.randomReadStream) this.add(2, this.randomStatelessFileDescriptorRead) this.add(1, this.createReadableFileDescriptor) - this.add(1, this.writeAndMkdir) + //this.add(1, this.writeAndMkdir) } // START Helper functions. @@ -250,7 +250,11 @@ class HyperdriveFuzzer extends FuzzBuzz { collect(stream, (err, bufs) => { if (err) return reject(err) let buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs) - if (!buf.equals(content.slice(start, start + length))) return reject(new Error('Read stream does not match content slice.')) + + if (!buf.equals(content.slice(start, start + length))) { + console.log('buf:', buf, 'content slice:', content.slice(start, start + length)) + return reject(new Error('Read stream does not match content slice.')) + } this.debug(`Random read stream for ${fileName} succeeded.`) return resolve() }) @@ -451,13 +455,13 @@ class SparseHyperdriveFuzzer extends HyperdriveFuzzer { async _setup () { await super._setup() - this.remoteDrive = create(this.drive.key) + this.remoteDrive = create(this.drive.key, { sparse: true }) return new Promise((resolve, reject) => { this.remoteDrive.ready(err => { if (err) throw err - let s1 = this.remoteDrive.replicate({ live: true }) - s1.pipe(this.drive.replicate({ live: true })).pipe(s1) + let s1 = this.remoteDrive.replicate({ live: true, encrypt: false }) + s1.pipe(this.drive.replicate({ live: true, encrypt: false })).pipe(s1) this.remoteDrive.ready(err => { if (err) return reject(err) return resolve() @@ -472,12 +476,12 @@ class SparseHyperdriveFuzzer extends HyperdriveFuzzer { module.exports = HyperdriveFuzzer -tape.only('20000 mixed operations, single drive', async t => { +tape('20000 mixed operations, single drive', async t => { t.plan(1) const fuzz = new HyperdriveFuzzer({ - seed: 'hyperdrive', - debugging: false + seed: 'hyperdrive2', + debugging: true }) try { @@ -488,12 +492,12 @@ tape.only('20000 mixed operations, single drive', async t => { } }) -tape('20000 mixed operations, replicating drives', async t => { +tape.only('20000 mixed operations, replicating drives', async t => { t.plan(1) const fuzz = new SparseHyperdriveFuzzer({ seed: 'hyperdrive2', - debugging: false + debugging: true }) try { diff --git a/test/index.js b/test/index.js index e1e1a3f5..c3a3787a 100644 --- a/test/index.js +++ b/test/index.js @@ -4,7 +4,7 @@ require('./creation') require('./deletion') // require('./diff') require('./fd') -// require('./fuzzing') +require('./fuzzing') require('./stat') require('./storage') require('./watch') From 229b4f9fcdc9fbbaed38cc3a94340482589b6ff3 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sat, 15 Jun 2019 18:58:37 +0200 Subject: [PATCH 074/108] Unskip fd tests and remove content update code --- index.js | 53 +++++++++++++++++++++++++++++++++++++------------ test/fd.js | 10 +++++----- test/fuzzing.js | 2 +- test/stat.js | 15 ++++++++------ 4 files changed, 55 insertions(+), 25 deletions(-) diff --git a/index.js b/index.js index cf670b1f..50cb1c27 100644 --- a/index.js +++ b/index.js @@ -167,18 +167,22 @@ class Hyperdrive extends EventEmitter { } _getContent (db, opts, cb) { + console.log('in _getContent') if (typeof opts === 'function') return this._getContent(db, null, opts) const self = this const existingContent = self._contentStates.get(db) if (existingContent) return process.nextTick(cb, null, existingContent) + console.log('calling getMetadata') db.getMetadata((err, publicKey) => { if (err) return cb(err) + console.log('got metadata, publicKey:', publicKey) return onkey(publicKey) }) function onkey (publicKey) { + console.log('in onkey') const contentOpts = { key: publicKey, ...contentOptions(self, opts && opts.secretKey), ...opts } const feed = self._corestore.get(contentOpts) feed.ready(err => { @@ -186,12 +190,6 @@ class Hyperdrive extends EventEmitter { const state = new ContentState(feed) self._contentStates.set(db, state) feed.on('error', err => self.emit('error', err)) - if (!feed.writable) { - return feed.update(1, err => { - if (err) return cb(err) - return cb(null, state) - }) - } return cb(null, state) }) } @@ -306,6 +304,21 @@ class Hyperdrive extends EventEmitter { console.log('st:', st) console.log('byteLength:', byteLength, 'blockOffset:', blockOffset, 'byteOffset:', byteOffset, 'blockLength:', blockLength) + /* + if (byteOffset === 104236) { + return feed.get(169, (err, contents) => { + console.log('CONTENTS AT BLOCK 169:', contents) + stream.start({ + feed, + blockOffset, + blockLength, + byteOffset, + byteLength + }) + }) + } + */ + stream.start({ feed, blockOffset, @@ -525,19 +538,29 @@ class Hyperdrive extends EventEmitter { } _createStat (name, opts, cb) { + const self = this + const statConstructor = (opts && opts.directory) ? Stat.directory : Stat.file this._db.get(name, (err, node, trie) => { if (err) return cb(err) - this._getContent(trie, (err, contentState) => { + onexisting(node, trie) + }) + + function onexisting (node, trie) { + self.ready(err => { if (err) return cb(err) - const st = statConstructor({ - ...opts, - offset: contentState.feed.length, - byteOffset: contentState.feed.byteLength + self._getContent(trie, (err, contentState) => { + if (err) return cb(err) + console.log(1) + const st = statConstructor({ + ...opts, + offset: contentState.feed.length, + byteOffset: contentState.feed.byteLength + }) + return cb(null, st) }) - return cb(null, st) }) - }) + } } mkdir (name, opts, cb) { @@ -549,8 +572,10 @@ class Hyperdrive extends EventEmitter { name = unixify(name) opts.directory = true + console.log('creating stat') this._createStat(name, opts, (err, st) => { if (err) return cb(err) + console.log('stat created, putting stat') this._putStat(name, st, { condition: ifNotExists }, cb) @@ -584,6 +609,7 @@ class Hyperdrive extends EventEmitter { }) function onstat (err, node, trie) { + console.log('IN ONSTAT, NODE:', node) if (err) return cb(err) if (!node && opts.trie) return cb(null, null, trie) if (!node && opts.file) return cb(new errors.FileNotFound(name)) @@ -597,6 +623,7 @@ class Hyperdrive extends EventEmitter { if (writingFd) { st.size = writingFd.stat.size } + console.log('HERE!!') cb(null, new Stat(st), trie) } } diff --git a/test/fd.js b/test/fd.js index e00dbeb4..4f87a3d8 100644 --- a/test/fd.js +++ b/test/fd.js @@ -195,7 +195,7 @@ tape('fd basic write, creating file', function (t) { }) }) -tape.skip('fd basic write, appending file', function (t) { +tape('fd basic write, appending file', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) @@ -225,7 +225,7 @@ tape.skip('fd basic write, appending file', function (t) { } }) -tape.skip('fd basic write, overwrite file', function (t) { +tape('fd basic write, overwrite file', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) @@ -255,7 +255,7 @@ tape.skip('fd basic write, overwrite file', function (t) { } }) -tape.skip('fd stateful write', function (t) { +tape('fd stateful write', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) @@ -280,7 +280,7 @@ tape.skip('fd stateful write', function (t) { }) }) -tape.skip('huge stateful write + stateless read', function (t) { +tape('huge stateful write + stateless read', function (t) { const NUM_SLICES = 1000 const SLICE_SIZE = 4096 const READ_SIZE = Math.floor(4096 * 2.75) @@ -336,7 +336,7 @@ tape.skip('huge stateful write + stateless read', function (t) { } }) -tape.skip('fd random-access write fails', function (t) { +tape('fd random-access write fails', function (t) { const drive = create() const content = Buffer.alloc(10000).fill('0123456789abcdefghijklmnopqrstuvwxyz') let first = content.slice(0, 2000) diff --git a/test/fuzzing.js b/test/fuzzing.js index cd0f42cf..ce82d61f 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -492,7 +492,7 @@ tape('20000 mixed operations, single drive', async t => { } }) -tape.only('20000 mixed operations, replicating drives', async t => { +tape.skip('20000 mixed operations, replicating drives', async t => { t.plan(1) const fuzz = new SparseHyperdriveFuzzer({ diff --git a/test/stat.js b/test/stat.js index 2afe3369..0a2fae24 100644 --- a/test/stat.js +++ b/test/stat.js @@ -4,11 +4,11 @@ var create = require('./helpers/create') var mask = 511 // 0b111111111 tape('stat file', function (t) { - var archive = create() + var drive = create() - archive.writeFile('/foo', 'bar', {mode: 438}, function (err) { + drive.writeFile('/foo', 'bar', {mode: 438}, function (err) { t.error(err, 'no error') - archive.stat('/foo', function (err, st) { + drive.stat('/foo', function (err, st) { t.error(err, 'no error') t.same(st.isDirectory(), false) t.same(st.isFile(), true) @@ -21,12 +21,15 @@ tape('stat file', function (t) { }) tape('stat dir', function (t) { - var archive = create() + var drive = create() - archive.mkdir('/foo', function (err) { + console.log('going into mkdir') + drive.mkdir('/foo', function (err) { + console.log('after mkdir') t.error(err, 'no error') - archive.stat('/foo', function (err, st) { + drive.stat('/foo', function (err, st) { t.error(err, 'no error') + console.log('right here') t.same(st.isDirectory(), true) t.same(st.isFile(), false) t.same(st.mode & mask, 493) From d4dee91ca0809f62e150ba779e49640b03cd1162 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 17 Jun 2019 13:31:20 +0200 Subject: [PATCH 075/108] Simple readdir with symlinks works --- index.js | 117 ++++++++++++++++++++---------------------------- lib/stat.js | 11 +++++ schema.proto | 27 ----------- test/fuzzing.js | 4 +- 4 files changed, 61 insertions(+), 98 deletions(-) delete mode 100644 schema.proto diff --git a/index.js b/index.js index 50cb1c27..9391c96c 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,6 @@ const path = require('path').posix const { EventEmitter } = require('events') +const { pipeline } = require('stream') const collect = require('stream-collector') const thunky = require('thunky') @@ -7,8 +8,6 @@ const unixify = require('unixify') const mutexify = require('mutexify') const duplexify = require('duplexify') const through = require('through2') -const pumpify = require('pumpify') -const pump = require('pump') const coreByteStream = require('hypercore-byte-stream') const MountableHypertrie = require('mountable-hypertrie') @@ -17,8 +16,8 @@ const createFileDescriptor = require('./lib/fd') const Stat = require('./lib/stat') const errors = require('./lib/errors') const defaultCorestore = require('./lib/storage') -const { messages } = require('hyperdrive-schemas') const { contentKeyPair, contentOptions, ContentState } = require('./lib/content') +const { createStatStream } = require('./lib/iterator') // 20 is arbitrary, just to make the fds > stdio etc const STDIO_CAP = 20 @@ -167,22 +166,18 @@ class Hyperdrive extends EventEmitter { } _getContent (db, opts, cb) { - console.log('in _getContent') if (typeof opts === 'function') return this._getContent(db, null, opts) const self = this const existingContent = self._contentStates.get(db) if (existingContent) return process.nextTick(cb, null, existingContent) - console.log('calling getMetadata') db.getMetadata((err, publicKey) => { if (err) return cb(err) - console.log('got metadata, publicKey:', publicKey) return onkey(publicKey) }) function onkey (publicKey) { - console.log('in onkey') const contentOpts = { key: publicKey, ...contentOptions(self, opts && opts.secretKey), ...opts } const feed = self._corestore.get(contentOpts) feed.ready(err => { @@ -198,7 +193,7 @@ class Hyperdrive extends EventEmitter { _putStat (name, stat, opts, cb) { if (typeof opts === 'function') return this._putStat(name, stat, null, opts) try { - var encoded = messages.Stat.encode(stat) + var encoded = stat.encode() } catch (err) { return cb(err) } @@ -209,13 +204,13 @@ class Hyperdrive extends EventEmitter { } _update (name, stat, cb) { - name = unixify(name) + name = fixName(name) this._db.get(name, (err, st) => { if (err) return cb(err) if (!st) return cb(new errors.FileNotFound(name)) try { - var decoded = messages.Stat.decode(st.value) + var decoded = Stat.decode(st.value) } catch (err) { return cb(err) } @@ -225,7 +220,7 @@ class Hyperdrive extends EventEmitter { } open (name, flags, cb) { - name = unixify(name) + name = fixName(name) this.ready(err => { if (err) return cb(err) @@ -264,7 +259,7 @@ class Hyperdrive extends EventEmitter { if (!opts) opts = {} const self = this - name = unixify(name) + name = fixName(name) const length = typeof opts.end === 'number' ? 1 + opts.end - (opts.start || 0) : typeof opts.length === 'number' ? opts.length : -1 const stream = coreByteStream({ @@ -364,37 +359,13 @@ class Hyperdrive extends EventEmitter { createDirectoryStream (name, opts) { if (!opts) opts = {} - - name = unixify(name) - - const proxy = duplexify.obj() - proxy.setWritable(false) - - this.ready(err => { - if (err) return - const stream = pump( - this._db.createReadStream(name, opts), - through.obj((chunk, enc, cb) => { - try { - var stat = messages.Stat.decode(chunk.value) - } catch (err) { - return cb(err) - } - return cb(null, { - path: chunk.key, - stat: new Stat(stat) - }) - }) - ) - proxy.setReadable(stream) - }) - - return proxy + name = fixName(name) + return createStatStream(this, this._db, name, opts) } createWriteStream (name, opts) { if (!opts) opts = {} - name = unixify(name) + name = fixName(name) const self = this var release @@ -462,7 +433,7 @@ class Hyperdrive extends EventEmitter { create (name, opts, cb) { if (typeof opts === 'function') return this.create(name, null, opts) - name = unixify(name) + name = fixName(name) this.ready(err => { if (err) return cb(err) @@ -480,7 +451,7 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'string') opts = {encoding: opts} if (!opts) opts = {} - name = unixify(name) + name = fixName(name) collect(this.createReadStream(name, opts), function (err, bufs) { if (err) return cb(err) @@ -496,7 +467,7 @@ class Hyperdrive extends EventEmitter { if (typeof buf === 'string') buf = Buffer.from(buf, opts.encoding || 'utf-8') if (!cb) cb = noop - name = unixify(name) + name = fixName(name) let stream = this.createWriteStream(name, opts) @@ -514,7 +485,7 @@ class Hyperdrive extends EventEmitter { } truncate (name, size, cb) { - name = unixify(name) + name = fixName(name) this.lstat(name, { file: true, trie: true }, (err, st) => { if (err && err.errno !== 2) return cb(err) @@ -551,7 +522,6 @@ class Hyperdrive extends EventEmitter { if (err) return cb(err) self._getContent(trie, (err, contentState) => { if (err) return cb(err) - console.log(1) const st = statConstructor({ ...opts, offset: contentState.feed.length, @@ -569,13 +539,11 @@ class Hyperdrive extends EventEmitter { if (!opts) opts = {} if (!cb) cb = noop - name = unixify(name) + name = fixName(name) opts.directory = true - console.log('creating stat') this._createStat(name, opts, (err, st) => { if (err) return cb(err) - console.log('stat created, putting stat') this._putStat(name, st, { condition: ifNotExists }, cb) @@ -589,7 +557,7 @@ class Hyperdrive extends EventEmitter { if (name !== '/' && !st) return cb(new errors.FileNotFound(name)) if (name === '/') return cb(null, Stat.directory()) try { - st = messages.Stat.decode(st.value) + st = Stat.decode(st.value) } catch (err) { return cb(err) } @@ -601,7 +569,7 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'function') return this.lstat(name, null, opts) if (!opts) opts = {} const self = this - name = unixify(name) + name = fixName(name) this.ready(err => { if (err) return cb(err) @@ -609,13 +577,12 @@ class Hyperdrive extends EventEmitter { }) function onstat (err, node, trie) { - console.log('IN ONSTAT, NODE:', node) if (err) return cb(err) if (!node && opts.trie) return cb(null, null, trie) if (!node && opts.file) return cb(new errors.FileNotFound(name)) if (!node) return self._statDirectory(name, opts, cb) try { - var st = messages.Stat.decode(node.value) + var st = Stat.decode(node.value) } catch (err) { return cb(err) } @@ -623,8 +590,7 @@ class Hyperdrive extends EventEmitter { if (writingFd) { st.size = writingFd.stat.size } - console.log('HERE!!') - cb(null, new Stat(st), trie) + cb(null, st, trie) } } @@ -643,7 +609,7 @@ class Hyperdrive extends EventEmitter { access (name, opts, cb) { if (typeof opts === 'function') return this.access(name, null, opts) if (!opts) opts = {} - name = unixify(name) + name = fixName(name) this.stat(name, opts, err => { cb(err) @@ -661,18 +627,25 @@ class Hyperdrive extends EventEmitter { readdir (name, opts, cb) { if (typeof opts === 'function') return this.readdir(name, null, opts) - - name = unixify(name) - if (name !== '/' && name.startsWith('/')) name = name.slice(1) + name = fixName(name) const recursive = !!(opts && opts.recursive) - this._db.list(name, { gt: true, recursive }, (err, list) => { + const nameStream = pipeline( + createStatStream(this, this._db, name, { ...opts, recursive }), + through.obj(({ path: statPath, stat }, enc, cb) => { + const relativePath = (name === statPath) ? statPath : path.relative(name, statPath) + console.log('NAME:', name, 'STATPATH:', statPath, 'RELATIVEPATH:', relativePath) + const splitPath = relativePath.split('/') + if (recursive) return cb(null, relativePath) + if (name === '/') return cb(null, splitPath[0]) + return cb(null, splitPath.length > 1 ? splitPath[1] : splitPath[0]) + }) + ) + return collect(nameStream, (err, entries) => { + console.log('COLLECTED ENTRIES:', entries) if (err) return cb(err) - return cb(null, list.map(st => { - if (name === '/') return st.key.split('/')[0] - return path.relative(name, st.key).split('/')[0] - })) + return cb(null, entries) }) } @@ -688,13 +661,13 @@ class Hyperdrive extends EventEmitter { } unlink (name, cb) { - name = unixify(name) + name = fixName(name) this._del(name, cb || noop) } rmdir (name, cb) { if (!cb) cb = noop - name = unixify(name) + name = fixName(name) const self = this @@ -744,7 +717,7 @@ class Hyperdrive extends EventEmitter { } watch (name, onchange) { - name = unixify(name) + name = fixName(name) return this._db.watch(name, onchange) } @@ -752,7 +725,7 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'function') return this.mount(path, key, null, opts) const self = this - path = unixify(path) + path = fixName(path) opts = opts || {} opts.mount = { @@ -782,14 +755,14 @@ class Hyperdrive extends EventEmitter { function mountCore () { self._createStat(path, opts, (err, st) => { if (err) return cb(err) - return self._db.put(path, messages.Stat.encode(st), cb) + return self._db.put(path, st.encode(), cb) }) } function mountTrie () { self._createStat(path, opts, (err, st) => { if (err) return cb(err) - self._db.mount(path, key, { ...opts, value: messages.Stat.encode(st) }, err => { + self._db.mount(path, key, { ...opts, value: st.encode() }, err => { if (err) return cb(err) return self._db.loadMount(path, cb) }) @@ -799,7 +772,7 @@ class Hyperdrive extends EventEmitter { symlink (target, linkName, cb) { target = unixify(target) - linkName = unixify(linkName) + linkName = fixName(linkName) this.lstat(linkName, (err, stat) => { if (err && (err.errno !== 2)) return cb(err) @@ -821,4 +794,10 @@ function ifNotExists (oldNode, newNode, cb) { return cb(null, true) } +function fixName (name) { + name = unixify(name) + if (!name.startsWith('/')) name = '/' + name + return name +} + function noop () {} diff --git a/lib/stat.js b/lib/stat.js index 997f4153..d6cf36ea 100644 --- a/lib/stat.js +++ b/lib/stat.js @@ -1,3 +1,5 @@ +const { messages: { Stat: StatEncoder } } = require('hyperdrive-schemas') + // http://man7.org/linux/man-pages/man2/stat.2.html var DEFAULT_FMODE = (4 | 2 | 0) << 6 | ((4 | 0 | 0) << 3) | (4 | 0 | 0) // rw-r--r-- var DEFAULT_DMODE = (4 | 2 | 1) << 6 | ((4 | 0 | 1) << 3) | (4 | 0 | 1) // rwxr-xr-x @@ -28,6 +30,10 @@ class Stat { return (mask & this.mode) === mask } + encode () { + return StatEncoder.encode(this) + } + isSocket () { return this._check(Stat.IFSOCK) } @@ -67,6 +73,11 @@ Stat.symlink = function (data) { return new Stat(data) } +Stat.decode = function (encodedStat) { + const st = StatEncoder.decode(encodedStat) + return new Stat(st) +} + Stat.IFSOCK = 0b1100 << 12 Stat.IFLNK = 0b1010 << 12 Stat.IFREG = 0b1000 << 12 diff --git a/schema.proto b/schema.proto deleted file mode 100644 index 824fc811..00000000 --- a/schema.proto +++ /dev/null @@ -1,27 +0,0 @@ -syntax = "proto2"; - -message Index { - required string type = 1; - optional bytes content = 2; -} - -message Mount { - required bytes key = 1; - optional uint64 version = 2; - optional bytes hash = 3; - optional bool hypercore = 4; -} - -message Stat { - required uint32 mode = 1; - optional uint32 uid = 2; - optional uint32 gid = 3; - optional uint64 size = 4; - optional uint64 blocks = 5; - optional uint64 offset = 6; - optional uint64 byteOffset = 7; - optional uint64 mtime = 8; - optional uint64 ctime = 9; - optional string linkname = 10; - optional Mount mount = 11; -} diff --git a/test/fuzzing.js b/test/fuzzing.js index ce82d61f..0061a965 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -480,8 +480,8 @@ tape('20000 mixed operations, single drive', async t => { t.plan(1) const fuzz = new HyperdriveFuzzer({ - seed: 'hyperdrive2', - debugging: true + seed: 'hyperdrive', + debugging: false }) try { From be2b24a59b0ea3559e3eac5e9a01fea20c18eb1b Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 17 Jun 2019 14:13:46 +0200 Subject: [PATCH 076/108] Statting symlinks works for links-to-links --- index.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 9391c96c..26bacf6b 100644 --- a/index.js +++ b/index.js @@ -598,11 +598,17 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'function') return this.stat(name, null, opts) if (!opts) opts = {} + console.log('STATTING:', name) this.lstat(name, opts, (err, stat, trie) => { if (err) return cb(err) - if (!stat) return cb(null, null, trie) - if (stat.linkname) return this.lstat(stat.linkname, opts, cb) - return cb(null, stat, trie) + if (!stat) return cb(null, null, trie, name) + if (stat.linkname) { + if (path.isAbsolute(stat.linkname)) return this.stat(stat.linkname, opts, cb) + const relativeStat = path.resolve('/', path.dirname(name), stat.linkname) + console.log('RELATIVE STAT:', relativeStat, 'NAME:', name, 'LINKNAME:', stat.linkname) + return this.stat(relativeStat, opts, cb) + } + return cb(null, stat, trie, name) }) } From 8bb81c20b00e9f40d21e2fe6ffcf40d2e6c342c2 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 17 Jun 2019 14:14:36 +0200 Subject: [PATCH 077/108] Added missing files --- lib/iterator.js | 89 ++++++++++++++++++ test/helpers/util.js | 13 +++ test/readdir.js | 218 +++++++++++++++++++++++++++++++++++++++++++ test/symlink.js | 100 ++++++++++++++++++++ 4 files changed, 420 insertions(+) create mode 100644 lib/iterator.js create mode 100644 test/helpers/util.js create mode 100644 test/readdir.js create mode 100644 test/symlink.js diff --git a/lib/iterator.js b/lib/iterator.js new file mode 100644 index 00000000..537d7c50 --- /dev/null +++ b/lib/iterator.js @@ -0,0 +1,89 @@ +const p = require('path').posix +const nanoiterator = require('nanoiterator') +const toStream = require('nanoiterator/to-stream') + +const Stat = require('./stat') + +function statIterator (drive, db, path, opts) { + const stack = [] + return nanoiterator({ open, next }) + + function open (cb) { + db.ready(err => { + if (err) return cb(err) + stack.unshift({ path: '/', target: null, iterator: db.iterator(path, opts) }) + return cb(null) + }) + } + + function next (cb) { + if (!stack.length) return cb(null, null) + stack[0].iterator.next((err, node) => { + if (err) return cb(err) + if (!node) { + stack.shift() + return next(cb) + } + + try { + var st = Stat.decode(node.value) + } catch (err) { + return cb(err) + } + + console.log('NEXT NODE:', node) + + if (st.linkname) { + if (p.isAbsolute(st.linkname)) { + var linkPath = st.linkname + } else { + linkPath = p.resolve('/', p.dirname(node.key), st.linkname) + } + console.log('HANDLING A LINK HERE, linkPath:', linkPath, 'path:', node.key, 'linkname:', st.linkname) + return pushLink(prefix(node.key), linkPath, (err, linkStat) => { + if (err) return cb(err) + console.log('linkStat:', linkStat) + if (linkStat) return cb(null, { stat: linkStat, path: prefix(node.key) }) + return next(cb) + }) + } + linkPath = stack[0].path + const resolved = (linkPath === '/') ? node.key : p.join(linkPath, node.key.slice(stack[0].target.length)) + console.log('RETURNING PATH:', resolved, 'STACK PATH:', stack[0].path, 'KEY:', node.key) + return cb(null, { stat: st, path: prefix(resolved) }) + }) + } + + function pushLink (nodePath, linkPath, cb) { + drive.stat(linkPath, (err, stat, _, resolvedLink) => { + console.log('GOT LINK INFO', linkPath, 'NODEPATH:', nodePath, 'STAT:', stat, 'RESOLVED LINK', resolvedLink) + if (!stat) return cb(null) + if (stat.isDirectory()) { + console.log('IT IS A DIRECTORY, NODEPATH:', nodePath, 'PATH:', path) + if (opts && opts.recursive || nodePath === path) { + stack.unshift({ path: nodePath, target: resolvedLink, iterator: db.iterator(resolvedLink, { gt: true, ...opts }) }) + console.log('UNSHIFTED ITERATOR FOR', 'path:', nodePath, 'target:', resolvedLink) + return cb(null) + } + return cb(null, { stat, path: linkPath}) + } + return cb(null, stat) + }) + } +} + +function createStatStream (drive, db, path, opts) { + const ite = statIterator(drive, db, path, opts) + return toStream(ite) +} + +function prefix (key) { + if (key.startsWith('/')) return key + return '/' + key +} + +module.exports = { + statIterator, + createStatStream +} + diff --git a/test/helpers/util.js b/test/helpers/util.js new file mode 100644 index 00000000..4cfd1720 --- /dev/null +++ b/test/helpers/util.js @@ -0,0 +1,13 @@ +module.exports.runAll = function (ops) { + return new Promise((resolve, reject) => { + runNext(ops.shift()) + function runNext (op) { + op(err => { + if (err) return reject(err) + let next = ops.shift() + if (!next) return resolve() + return runNext(next) + }) + } + }) +} diff --git a/test/readdir.js b/test/readdir.js new file mode 100644 index 00000000..bd6346e6 --- /dev/null +++ b/test/readdir.js @@ -0,0 +1,218 @@ +const crypto = require('crypto') +const test = require('tape') +const collect = require('stream-collector') + +const create = require('./helpers/create') +const { runAll } = require('./helpers/util') + +test('simple readdir', async t => { + const drive = create() + + const files = createFiles([ + 'a/a', + 'a/b', + 'a/c/d', + 'a/c/e', + 'a/e', + 'b/e', + 'b/f', + 'b/d', + 'e' + ]) + + try { + await runAll([ + cb => writeFiles(drive, files, cb), + cb => validateReaddir(t, drive, 'a', ['a', 'b', 'c', 'e'], cb), + cb => validateReaddir(t, drive, 'a/c', ['d', 'e'], cb), + cb => validateReaddir(t, drive, 'b', ['e', 'f', 'd'], cb), + cb => validateReaddir(t, drive, '', ['a', 'b', 'e'], cb) + ]) + } catch (err) { + t.fail(err) + } + + t.end() +}) + +test('recursive readdir', async t => { + const drive = create() + + const files = createFiles([ + 'a/a', + 'a/b', + 'a/c/d', + 'a/c/e', + 'a/e', + 'b/e', + 'b/f', + 'b/d', + 'e' + ]) + + try { + await runAll([ + cb => writeFiles(drive, files, cb), + cb => validateReaddir(t, drive, 'a', ['a', 'b', 'c/d', 'c/e', 'e'], { recursive: true }, cb), + cb => validateReaddir(t, drive, 'a/c', ['d', 'e'], { recursive: true }, cb), + cb => validateReaddir(t, drive, 'b', ['e', 'f', 'd'], { recursive: true }, cb), + cb => validateReaddir(t, drive, '', ['a/a', 'a/b', 'a/c/d', 'a/c/e', 'a/e', 'b/e', 'b/f', 'b/d', 'e'], { recursive: true }, cb) + ]) + } catch (err) { + t.fail(err) + } + + t.end() +}) + +test('readdir follows symlink', async t => { + const drive = create() + + const files = createFiles([ + 'a/a', + 'a/b', + 'a/c/d', + 'a/c/e', + 'a/e', + 'b/e', + 'b/f', + 'b/d', + 'e' + ]) + const links = new Map([ + ['f', 'a'], + ['p', 'a/c'], + ['g', 'e'] + ]) + + try { + await runAll([ + cb => writeFiles(drive, files, cb), + cb => writeLinks(drive, links, cb), + cb => validateReaddir(t, drive, 'f', ['a', 'b', 'c', 'e'], cb), + cb => validateReaddir(t, drive, 'p', ['d', 'e'], cb), + cb => validateReaddir(t, drive, 'b', ['e', 'f', 'd'], cb), + cb => validateReaddir(t, drive, '', ['a', 'b', 'e', 'f', 'p', 'g'], cb) + ]) + } catch (err) { + t.fail(err) + } + + t.end() +}) + +test('readdir follows symlink', async t => { + const drive = create() + + const files = createFiles([ + 'a/a', + 'a/b', + 'a/c/d', + 'a/c/e', + 'a/e', + 'b/e', + 'b/f', + 'b/d', + 'e' + ]) + const links = new Map([ + ['f', 'a'], + ['p', 'a/c'], + ['g', 'e'] + ]) + + const fExpected = ['f/b', 'f/c/d', 'f/c/e', 'f/e', 'f/a'] + const pExpected = ['p/e', 'p/d'] + const rootExpected = ['a/a', 'a/b', 'a/c/d', 'a/c/e', 'a/e', 'b/e', 'b/f', 'b/d', 'e', 'g'] + + try { + await runAll([ + cb => writeFiles(drive, files, cb), + cb => writeLinks(drive, links, cb), + cb => validateReaddir(t, drive, 'f', ['a', 'b', 'c/d', 'c/e', 'e'], { recursive: true }, cb), + cb => validateReaddir(t, drive, 'p', ['d', 'e'], { recursive: true }, cb), + cb => validateReaddir(t, drive, 'b', ['e', 'f', 'd'], { recursive: true }, cb), + cb => validateReaddir(t, drive, '', [...rootExpected, ...fExpected, ...pExpected], { recursive: true }, cb) + ]) + } catch (err) { + t.fail(err) + } + + t.end() +}) + +test('readdir follows symlinks to symlinks', async t => { + const drive = create() + + const files = createFiles([ + 'a/a', + 'a/b', + 'a/c/d', + 'a/c/e', + 'a/e', + 'b/e', + 'b/f', + 'b/d', + 'e' + ]) + const links = new Map([ + ['a/d', '../r'], + ['r', 'a/c/f'], + ['a/c/f', '../../b'] + ]) + + try { + await runAll([ + cb => writeFiles(drive, files, cb), + cb => writeLinks(drive, links, cb), + cb => validateReaddir(t, drive, 'a/d', ['e', 'f', 'd'], cb), + cb => validateReaddir(t, drive, 'r', ['e', 'f', 'd'], cb), + cb => validateReaddir(t, drive, 'a/c/f', ['e', 'f', 'd'], cb), + cb => validateReaddir(t, drive, '', ['a', 'b', 'e', 'r'], cb) + ]) + } catch (err) { + t.fail(err) + } + + t.end() +}) + +function validateReaddir (t, drive, path, names, opts, cb) { + if (typeof opts === 'function') return validateReaddir(t, drive, path, names, {}, opts) + drive.readdir(path, opts, (err, list) => { + if (err) return cb(err) + t.same(list.length, names.length) + for (const name of list) { + t.notEqual(names.indexOf(name), -1) + } + return cb(null) + }) +} + +function writeFiles (drive, files, cb) { + var expected = files.size + for (const [name, contents] of files) { + drive.writeFile(name, contents, err => { + if (err) return cb(err) + if (!--expected) return cb(null) + }) + } +} + +function writeLinks (drive, links, cb) { + var expected = links.size + for (const [name, target] of links) { + drive.symlink(target, name, err => { + if (err) return cb(err) + if (!--expected) return cb(null) + }) + } +} + +function createFiles (names) { + const files = [] + for (const name of names) { + files.push([name, crypto.randomBytes(32)]) + } + return new Map(files) +} diff --git a/test/symlink.js b/test/symlink.js new file mode 100644 index 00000000..2aa66e1d --- /dev/null +++ b/test/symlink.js @@ -0,0 +1,100 @@ +var test = require('tape') +var create = require('./helpers/create') + +test('basic symlink', t => { + const archive = create() + + archive.writeFile('/hello.txt', 'world', err => { + t.error(err, 'no error') + archive.symlink('/hello.txt', '/link.txt', err => { + t.error(err, 'no error') + onlink() + }) + }) + + function onlink () { + archive.stat('/link.txt', (err, st) => { + t.error(err, 'no error') + t.same(st.size, 5) + archive.readFile('/link.txt', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('world')) + t.end() + }) + }) + } +}) + +test('fixing a broken symlink', t => { + const archive = create() + + archive.symlink('/hello.txt', '/link.txt', err => { + t.error(err, 'no error') + archive.stat('/link.txt', (err, st) => { + t.same(err.errno, 2) + archive.writeFile('/hello.txt', 'world', err => { + t.error(err, 'no error') + onwrite() + }) + }) + }) + + function onwrite () { + archive.stat('/link.txt', (err, st) => { + t.error(err, 'no error') + t.same(st.size, 5) + archive.readFile('/link.txt', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('world')) + t.end() + }) + }) + } +}) + +test('unlinking a symlink does not delete the target', t => { + const archive = create() + + archive.writeFile('/hello.txt', 'world', err => { + t.error(err, 'no error') + archive.symlink('/hello.txt', '/link.txt', err => { + t.error(err, 'no error') + archive.unlink('/link.txt', err => { + t.error(err, 'no error') + onunlink() + }) + }) + }) + + function onunlink () { + archive.stat('/hello.txt', (err, st) => { + t.error(err, 'no error') + t.same(st.size, 5) + archive.readFile('/hello.txt', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('world')) + t.end() + }) + }) + } +}) + +test('symlinks appear in readdir', t => { + const archive = create() + + archive.writeFile('/hello.txt', 'world', err => { + t.error(err, 'no error') + archive.symlink('/hello.txt', '/link.txt', err => { + t.error(err, 'no error') + onlink() + }) + }) + + function onlink () { + archive.readdir('/', (err, files) => { + t.error(err, 'no errors') + t.same(files, ['hello.txt', 'link.txt']) + t.end() + }) + } +}) From 2abcf7a1002a6de3c5bd33c68e60c828462ce116 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 17 Jun 2019 14:28:23 +0200 Subject: [PATCH 078/108] Removed debugging statements --- index.js | 28 +----------- lib/iterator.js | 8 ---- test/basic.js | 48 +-------------------- test/readdir.js | 112 +++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 114 insertions(+), 82 deletions(-) diff --git a/index.js b/index.js index 26bacf6b..f139b66d 100644 --- a/index.js +++ b/index.js @@ -8,6 +8,7 @@ const unixify = require('unixify') const mutexify = require('mutexify') const duplexify = require('duplexify') const through = require('through2') +const pump = require('pump') const coreByteStream = require('hypercore-byte-stream') const MountableHypertrie = require('mountable-hypertrie') @@ -279,7 +280,6 @@ class Hyperdrive extends EventEmitter { }) function oncontent (st, contentState) { - console.log('IN ONCONTENT, content length:', contentState.feed.length, 'byteLength:', contentState.feed.byteLength) if (st.mount && st.mount.hypercore) { var byteOffset = 0 var blockOffset = 0 @@ -296,24 +296,6 @@ class Hyperdrive extends EventEmitter { } const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) - console.log('st:', st) - console.log('byteLength:', byteLength, 'blockOffset:', blockOffset, 'byteOffset:', byteOffset, 'blockLength:', blockLength) - - /* - if (byteOffset === 104236) { - return feed.get(169, (err, contents) => { - console.log('CONTENTS AT BLOCK 169:', contents) - stream.start({ - feed, - blockOffset, - blockLength, - byteOffset, - byteLength - }) - }) - } - */ - stream.start({ feed, blockOffset, @@ -598,14 +580,12 @@ class Hyperdrive extends EventEmitter { if (typeof opts === 'function') return this.stat(name, null, opts) if (!opts) opts = {} - console.log('STATTING:', name) this.lstat(name, opts, (err, stat, trie) => { if (err) return cb(err) if (!stat) return cb(null, null, trie, name) if (stat.linkname) { if (path.isAbsolute(stat.linkname)) return this.stat(stat.linkname, opts, cb) const relativeStat = path.resolve('/', path.dirname(name), stat.linkname) - console.log('RELATIVE STAT:', relativeStat, 'NAME:', name, 'LINKNAME:', stat.linkname) return this.stat(relativeStat, opts, cb) } return cb(null, stat, trie, name) @@ -641,15 +621,11 @@ class Hyperdrive extends EventEmitter { createStatStream(this, this._db, name, { ...opts, recursive }), through.obj(({ path: statPath, stat }, enc, cb) => { const relativePath = (name === statPath) ? statPath : path.relative(name, statPath) - console.log('NAME:', name, 'STATPATH:', statPath, 'RELATIVEPATH:', relativePath) - const splitPath = relativePath.split('/') if (recursive) return cb(null, relativePath) - if (name === '/') return cb(null, splitPath[0]) - return cb(null, splitPath.length > 1 ? splitPath[1] : splitPath[0]) + return cb(null, relativePath.split('/')[0]) }) ) return collect(nameStream, (err, entries) => { - console.log('COLLECTED ENTRIES:', entries) if (err) return cb(err) return cb(null, entries) }) diff --git a/lib/iterator.js b/lib/iterator.js index 537d7c50..9e8967e2 100644 --- a/lib/iterator.js +++ b/lib/iterator.js @@ -31,38 +31,30 @@ function statIterator (drive, db, path, opts) { return cb(err) } - console.log('NEXT NODE:', node) - if (st.linkname) { if (p.isAbsolute(st.linkname)) { var linkPath = st.linkname } else { linkPath = p.resolve('/', p.dirname(node.key), st.linkname) } - console.log('HANDLING A LINK HERE, linkPath:', linkPath, 'path:', node.key, 'linkname:', st.linkname) return pushLink(prefix(node.key), linkPath, (err, linkStat) => { if (err) return cb(err) - console.log('linkStat:', linkStat) if (linkStat) return cb(null, { stat: linkStat, path: prefix(node.key) }) return next(cb) }) } linkPath = stack[0].path const resolved = (linkPath === '/') ? node.key : p.join(linkPath, node.key.slice(stack[0].target.length)) - console.log('RETURNING PATH:', resolved, 'STACK PATH:', stack[0].path, 'KEY:', node.key) return cb(null, { stat: st, path: prefix(resolved) }) }) } function pushLink (nodePath, linkPath, cb) { drive.stat(linkPath, (err, stat, _, resolvedLink) => { - console.log('GOT LINK INFO', linkPath, 'NODEPATH:', nodePath, 'STAT:', stat, 'RESOLVED LINK', resolvedLink) if (!stat) return cb(null) if (stat.isDirectory()) { - console.log('IT IS A DIRECTORY, NODEPATH:', nodePath, 'PATH:', path) if (opts && opts.recursive || nodePath === path) { stack.unshift({ path: nodePath, target: resolvedLink, iterator: db.iterator(resolvedLink, { gt: true, ...opts }) }) - console.log('UNSHIFTED ITERATOR FOR', 'path:', nodePath, 'target:', resolvedLink) return cb(null) } return cb(null, { stat, path: linkPath}) diff --git a/test/basic.js b/test/basic.js index 878d2de3..e02a2b4b 100644 --- a/test/basic.js +++ b/test/basic.js @@ -157,7 +157,7 @@ tape('can read a single directory', async function (t) { } }) -tape('can stream a large directory', async function (t) { +tape.skip('can stream a large directory', async function (t) { const drive = create(null) let files = new Array(1000).fill(0).map((_, idx) => '' + idx) @@ -189,52 +189,6 @@ tape('can stream a large directory', async function (t) { } }) -tape('can read nested directories', async function (t) { - const drive = create(null) - - let files = ['a', 'b/a/b', 'b/c', 'c/b', 'd/e/f/g/h', 'd/e/a', 'e/a', 'e/b', 'f', 'g'] - let rootSet = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g']) - let bSet = new Set(['a', 'c']) - let dSet = new Set(['e']) - let eSet = new Set(['a', 'b']) - let deSet = new Set(['f', 'a']) - - for (let file of files) { - await insertFile(file, 'a small file') - } - - await checkDir('/', rootSet) - await checkDir('b', bSet) - await checkDir('d', dSet) - await checkDir('e', eSet) - await checkDir('d/e', deSet) - - t.end() - - function checkDir (dir, fileSet) { - return new Promise(resolve => { - drive.readdir(dir, (err, files) => { - t.error(err, 'no error') - for (let file of files) { - t.true(fileSet.has(file), 'correct file was listed') - fileSet.delete(file) - } - t.same(fileSet.size, 0, 'all files were listed') - return resolve() - }) - }) - } - - function insertFile (name, content) { - return new Promise((resolve, reject) => { - drive.writeFile(name, content, err => { - if (err) return reject(err) - return resolve() - }) - }) - } -}) - tape('can read sparse metadata', async function (t) { const { read, write } = await getTestDrives() diff --git a/test/readdir.js b/test/readdir.js index bd6346e6..ef14fddf 100644 --- a/test/readdir.js +++ b/test/readdir.js @@ -5,7 +5,37 @@ const collect = require('stream-collector') const create = require('./helpers/create') const { runAll } = require('./helpers/util') -test('simple readdir', async t => { +test('can read a single directory', async function (t) { + const drive = create(null) + + let files = ['a', 'b', 'c', 'd', 'e', 'f'] + let fileSet = new Set(files) + + for (let file of files) { + await insertFile(file, 'a small file') + } + + drive.readdir('/', (err, files) => { + t.error(err, 'no error') + for (let file of files) { + t.true(fileSet.has(file), 'correct file was listed') + fileSet.delete(file) + } + t.same(fileSet.size, 0, 'all files were listed') + t.end() + }) + + function insertFile (name, content) { + return new Promise((resolve, reject) => { + drive.writeFile(name, content, err => { + if (err) return reject(err) + return resolve() + }) + }) + } +}) + +test('another single-directory readdir', async t => { const drive = create() const files = createFiles([ @@ -177,6 +207,86 @@ test('readdir follows symlinks to symlinks', async t => { t.end() }) +test('can read nested directories', async function (t) { + const drive = create(null) + + let files = ['a', 'b/a/b', 'b/c', 'c/b', 'd/e/f/g/h', 'd/e/a', 'e/a', 'e/b', 'f', 'g'] + let rootSet = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g']) + let bSet = new Set(['a', 'c']) + let dSet = new Set(['e']) + let eSet = new Set(['a', 'b']) + let deSet = new Set(['f', 'a']) + + for (let file of files) { + await insertFile(file, 'a small file') + } + + await checkDir('/', rootSet) + await checkDir('b', bSet) + await checkDir('d', dSet) + await checkDir('e', eSet) + await checkDir('d/e', deSet) + + t.end() + + function checkDir (dir, fileSet) { + return new Promise(resolve => { + drive.readdir(dir, (err, files) => { + t.error(err, 'no error') + for (let file of files) { + t.true(fileSet.has(file), 'correct file was listed') + fileSet.delete(file) + } + t.same(fileSet.size, 0, 'all files were listed') + return resolve() + }) + }) + } + + function insertFile (name, content) { + return new Promise((resolve, reject) => { + drive.writeFile(name, content, err => { + if (err) return reject(err) + return resolve() + }) + }) + } +}) + +test('can stream a large directory', async function (t) { + const drive = create(null) + + let files = new Array(1000).fill(0).map((_, idx) => '/' + idx) + let fileSet = new Set(files) + + for (let file of files) { + await insertFile(file, 'a small file') + } + + let stream = drive.createDirectoryStream('/') + stream.on('data', ({ path, stat }) => { + if (!fileSet.has(path)) { + return t.fail('an incorrect file was streamed') + } + fileSet.delete(path) + }) + stream.on('end', () => { + t.same(fileSet.size, 0, 'all files were streamed') + t.end() + }) + + function insertFile (name, content) { + return new Promise((resolve, reject) => { + drive.writeFile(name, content, err => { + if (err) return reject(err) + return resolve() + }) + }) + } +}) + + + function validateReaddir (t, drive, path, names, opts, cb) { if (typeof opts === 'function') return validateReaddir(t, drive, path, names, {}, opts) drive.readdir(path, opts, (err, list) => { From 6feb8c4e6a8a111cd2456cb57d759f5f626c805c Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 17 Jun 2019 14:39:26 +0200 Subject: [PATCH 079/108] Update deps --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 422c1341..d79643bd 100644 --- a/package.json +++ b/package.json @@ -27,11 +27,11 @@ "filesystem-constants": "^1.0.0", "hypercore-byte-stream": "^1.0.2", "hyperdrive-schemas": "^0.9.0", - "mountable-hypertrie": "git+https://github.com/andrewosh/mountable-hypertrie#master", + "mountable-hypertrie": "^0.9.0", "mutexify": "^1.2.0", "pump": "^3.0.0", "pumpify": "^1.5.1", - "random-access-corestore": "git+https://github.com/andrewosh/random-access-corestore#master", + "random-access-corestore": "^0.9.0", "sodium-universal": "^2.0.0", "stream-collector": "^1.0.1", "through2": "^3.0.0", @@ -44,6 +44,6 @@ "random-access-memory": "^3.1.1", "tape": "^4.10.0", "temporary-directory": "^1.0.2", - "megastore": "git+https://github.com/andrewosh/megastore" + "megastore": "^0.9.0" } } From c2c4ac8700e926821050fc8fe95e4c86eb210b2c Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 17 Jun 2019 14:46:06 +0200 Subject: [PATCH 080/108] Bump corestore version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d79643bd..26ad0025 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "mutexify": "^1.2.0", "pump": "^3.0.0", "pumpify": "^1.5.1", - "random-access-corestore": "^0.9.0", + "random-access-corestore": "^0.9.1", "sodium-universal": "^2.0.0", "stream-collector": "^1.0.1", "through2": "^3.0.0", From a1832e290e6e666cb26d205da3371e64ed2d6c9d Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 17 Jun 2019 15:45:13 +0200 Subject: [PATCH 081/108] Re-enabled sparse-mode fuzz test --- test/fuzzing.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/fuzzing.js b/test/fuzzing.js index 0061a965..c4b68890 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -492,12 +492,12 @@ tape('20000 mixed operations, single drive', async t => { } }) -tape.skip('20000 mixed operations, replicating drives', async t => { +tape('20000 mixed operations, replicating drives', async t => { t.plan(1) const fuzz = new SparseHyperdriveFuzzer({ seed: 'hyperdrive2', - debugging: true + debugging: false }) try { From e8b76eb9049815761433bcec3a93cca7d6dc8366 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 17 Jun 2019 15:54:44 +0200 Subject: [PATCH 082/108] pipeline -> pump for node 8 --- index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/index.js b/index.js index f139b66d..3e667bc0 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,5 @@ const path = require('path').posix const { EventEmitter } = require('events') -const { pipeline } = require('stream') const collect = require('stream-collector') const thunky = require('thunky') @@ -617,7 +616,7 @@ class Hyperdrive extends EventEmitter { const recursive = !!(opts && opts.recursive) - const nameStream = pipeline( + const nameStream = pump( createStatStream(this, this._db, name, { ...opts, recursive }), through.obj(({ path: statPath, stat }, enc, cb) => { const relativePath = (name === statPath) ? statPath : path.relative(name, statPath) From 13d5227b39d09175174df56d87182aa2c4d148ea Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 17 Jun 2019 17:18:19 +0200 Subject: [PATCH 083/108] Fix readdir bug --- index.js | 1 + test/readdir.js | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/index.js b/index.js index 3e667bc0..08a2e7e8 100644 --- a/index.js +++ b/index.js @@ -620,6 +620,7 @@ class Hyperdrive extends EventEmitter { createStatStream(this, this._db, name, { ...opts, recursive }), through.obj(({ path: statPath, stat }, enc, cb) => { const relativePath = (name === statPath) ? statPath : path.relative(name, statPath) + if (relativePath === name) return cb(null) if (recursive) return cb(null, relativePath) return cb(null, relativePath.split('/')[0]) }) diff --git a/test/readdir.js b/test/readdir.js index ef14fddf..215ccc3f 100644 --- a/test/readdir.js +++ b/test/readdir.js @@ -5,6 +5,35 @@ const collect = require('stream-collector') const create = require('./helpers/create') const { runAll } = require('./helpers/util') +test('readdir on empty directory', async function (t) { + const drive = create() + + const files = createFiles([ + 'a/a', + 'a/b', + 'a/c/d', + 'a/c/e', + 'a/e', + 'b/e', + 'b/f', + 'b/d', + 'e' + ]) + + try { + await runAll([ + cb => drive.mkdir('l', cb), + cb => writeFiles(drive, files, cb), + cb => validateReaddir(t, drive, 'd', [], cb), + cb => validateReaddir(t, drive, 'l' ,[], cb) + ]) + } catch (err) { + t.fail(err) + } + + t.end() +}) + test('can read a single directory', async function (t) { const drive = create(null) From 1e8754472670e2bbad9ea1bcb89deb87bb8099db Mon Sep 17 00:00:00 2001 From: Mathias Buus Date: Tue, 18 Jun 2019 15:26:30 +0200 Subject: [PATCH 084/108] do less seeks (#239) --- index.js | 7 ++++--- test/basic.js | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index b032e2a7..cfd8e83d 100644 --- a/index.js +++ b/index.js @@ -301,15 +301,16 @@ class Hyperdrive extends EventEmitter { return stream.destroy(err) } - const byteOffset = opts.start ? st.byteOffset + opts.start : st.byteOffset - const byteLength = length !== -1 ? length : (opts.start ? st.size - opts.start : st.size) + const byteLength = length + const byteOffset = opts.start ? st.byteOffset + opts.start : (length === -1 ? -1 : st.byteOffset) stream.start({ feed: this.content, blockOffset: st.offset, blockLength: st.blocks, byteOffset, - byteLength + byteLength, + stat: st }) }) }) diff --git a/test/basic.js b/test/basic.js index 2ee73320..c4f09fa8 100644 --- a/test/basic.js +++ b/test/basic.js @@ -7,7 +7,6 @@ tape('write and read', function (t) { archive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') - console.log('reading') archive.readFile('/hello.txt', function (err, buf) { t.error(err, 'no error') t.same(buf, Buffer.from('world')) From faf6280977cd9667f8d780a55ab6edd138c2cc27 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Wed, 19 Jun 2019 02:21:35 +0200 Subject: [PATCH 085/108] rmdir should use a gt iterator --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index dfc11c7b..df23d434 100644 --- a/index.js +++ b/index.js @@ -654,7 +654,7 @@ class Hyperdrive extends EventEmitter { const self = this - let stream = this._db.iterator(name) + let stream = this._db.iterator(name, { gt: true }) stream.next((err, val) => { if (err) return cb(err) if (val) return cb(new errors.DirectoryNotEmpty(name)) From 26a89d9f622853e69c81739ed14076bb3700fad0 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 20 Jun 2019 02:27:40 +0200 Subject: [PATCH 086/108] Re-add sparseMetadata --- index.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index df23d434..ace21c55 100644 --- a/index.js +++ b/index.js @@ -41,10 +41,9 @@ class Hyperdrive extends EventEmitter { this.live = true this.sparse = opts.sparse !== false // TODO: Add support for mixed-sparsity. - this.sparseMetadata = this.sparse || opts.sparseMetadata !== false + this.sparseMetadata = opts.sparseMetadata !== false this._corestore = defaultCorestore(storage, { - sparse: this.sparse, valueEncoding: 'binary' }) this.metadata = this._corestore.default({ @@ -53,7 +52,7 @@ class Hyperdrive extends EventEmitter { }) this._db = opts._db || new MountableHypertrie(this._corestore, key, { feed: this.metadata, - sparse: this.sparse + sparse: this.sparseMetadata }) this._contentStates = new Map() @@ -179,7 +178,7 @@ class Hyperdrive extends EventEmitter { function onkey (publicKey) { const contentOpts = { key: publicKey, ...contentOptions(self, opts && opts.secretKey), ...opts } - const feed = self._corestore.get(contentOpts) + const feed = self._corestore.get({ ...contentOpts, sparse: self.sparse }) feed.ready(err => { if (err) return cb(err) const state = new ContentState(feed) From 2bda2cba30cfba8dffffed38ae4084587bb211e9 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 20 Jun 2019 17:41:36 +0200 Subject: [PATCH 087/108] Another mount test + debugging --- index.js | 8 +++++--- lib/iterator.js | 1 + test/mount.js | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index ace21c55..d34a0182 100644 --- a/index.js +++ b/index.js @@ -40,19 +40,20 @@ class Hyperdrive extends EventEmitter { this.discoveryKey = null this.live = true this.sparse = opts.sparse !== false - // TODO: Add support for mixed-sparsity. this.sparseMetadata = opts.sparseMetadata !== false + console.error('HYPERDRIVE SPARSE:', this.sparse, 'SPARSE METADATA:', this.sparseMetadata) + this._corestore = defaultCorestore(storage, { valueEncoding: 'binary' }) this.metadata = this._corestore.default({ key, + sparse: this.sparseMetadata, secretKey: opts.secretKey, }) this._db = opts._db || new MountableHypertrie(this._corestore, key, { feed: this.metadata, - sparse: this.sparseMetadata }) this._contentStates = new Map() @@ -178,9 +179,10 @@ class Hyperdrive extends EventEmitter { function onkey (publicKey) { const contentOpts = { key: publicKey, ...contentOptions(self, opts && opts.secretKey), ...opts } - const feed = self._corestore.get({ ...contentOpts, sparse: self.sparse }) + const feed = self._corestore.get(contentOpts) feed.ready(err => { if (err) return cb(err) + console.error('HYPERCORE FEED HERE:', feed) const state = new ContentState(feed) self._contentStates.set(db, state) feed.on('error', err => self.emit('error', err)) diff --git a/lib/iterator.js b/lib/iterator.js index 9e8967e2..38ad0ca0 100644 --- a/lib/iterator.js +++ b/lib/iterator.js @@ -6,6 +6,7 @@ const Stat = require('./stat') function statIterator (drive, db, path, opts) { const stack = [] + return nanoiterator({ open, next }) function open (cb) { diff --git a/test/mount.js b/test/mount.js index ec87c206..cdd1940a 100644 --- a/test/mount.js +++ b/test/mount.js @@ -205,6 +205,40 @@ test('cross-mount symlink', t => { } }) +test('lists nested mounts, shared write capabilities', async t => { + const megastore = new Megastore(ram, memdb(), false) + await megastore.ready() + + const cs1 = megastore.get('cs1') + const cs2 = megastore.get('cs2') + const cs3 = megastore.get('cs3') + + const drive1 = create({ corestore: cs1 }) + const drive2 = create({ corestore: cs2 }) + const drive3 = create({ corestore: cs3 }) + + drive3.ready(err => { + t.error(err, 'no error') + drive1.mount('a', drive2.key, err => { + t.error(err, 'no error') + drive1.mount('a/b', drive3.key, err => { + t.error(err, 'no error') + onmount() + }) + }) + }) + + function onmount () { + drive2.lstat('b', (err, stat) => { + drive1.readdir('a', (err, list) => { + t.error(err, 'no error') + t.same(list, ['b']) + t.end() + }) + }) + } +}) + test('dynamically resolves cross-mount symlinks') test('symlinks cannot break the sandbox') From 79f4be58f282fa7eda972d00c80b6e84b7d50127 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 21 Jun 2019 00:51:56 +0200 Subject: [PATCH 088/108] Update deps + don't forward top-level options into corestore --- index.js | 6 ++-- package.json | 7 +++-- test/basic.js | 18 ++++++++++++ test/mount.js | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index d34a0182..24fdea67 100644 --- a/index.js +++ b/index.js @@ -42,8 +42,6 @@ class Hyperdrive extends EventEmitter { this.sparse = opts.sparse !== false this.sparseMetadata = opts.sparseMetadata !== false - console.error('HYPERDRIVE SPARSE:', this.sparse, 'SPARSE METADATA:', this.sparseMetadata) - this._corestore = defaultCorestore(storage, { valueEncoding: 'binary' }) @@ -54,6 +52,7 @@ class Hyperdrive extends EventEmitter { }) this._db = opts._db || new MountableHypertrie(this._corestore, key, { feed: this.metadata, + sparse: this.sparseMetadata }) this._contentStates = new Map() @@ -178,11 +177,10 @@ class Hyperdrive extends EventEmitter { }) function onkey (publicKey) { - const contentOpts = { key: publicKey, ...contentOptions(self, opts && opts.secretKey), ...opts } + const contentOpts = { key: publicKey, ...contentOptions(self, opts && opts.secretKey) } const feed = self._corestore.get(contentOpts) feed.ready(err => { if (err) return cb(err) - console.error('HYPERCORE FEED HERE:', feed) const state = new ContentState(feed) self._contentStates.set(db, state) feed.on('error', err => self.emit('error', err)) diff --git a/package.json b/package.json index 26ad0025..3da5ec92 100644 --- a/package.json +++ b/package.json @@ -40,10 +40,13 @@ }, "devDependencies": { "fuzzbuzz": "^2.0.0", + "megastore": "^0.9.0", + "megastore-swarm-networking": "^1.0.1", "memdb": "^1.3.1", + "random-access-file": "^2.1.3", "random-access-memory": "^3.1.1", + "rimraf": "^2.6.3", "tape": "^4.10.0", - "temporary-directory": "^1.0.2", - "megastore": "^0.9.0" + "temporary-directory": "^1.0.2" } } diff --git a/test/basic.js b/test/basic.js index e02a2b4b..0f5a3f5d 100644 --- a/test/basic.js +++ b/test/basic.js @@ -232,3 +232,21 @@ tape('can read sparse metadata', async function (t) { }) } }) + +tape('unavailable drive becomes ready', function (t) { + var drive1 = create() + var drive2 = null + + drive1.ready(err => { + t.error(err, 'no error') + drive2 = create(drive1.key) + drive2.ready(err => { + t.error(err, 'no error') + drive2.readFile('blah', (err, contents) => { + t.true(err) + t.same(err.errno, 2) + t.end() + }) + }) + }) +}) diff --git a/test/mount.js b/test/mount.js index cdd1940a..487d86c3 100644 --- a/test/mount.js +++ b/test/mount.js @@ -1,8 +1,13 @@ var test = require('tape') const ram = require('random-access-memory') +const raf = require('random-access-file') const memdb = require('memdb') +const rimraf = require('rimraf') + const corestore = require('random-access-corestore') const Megastore = require('megastore') +const SwarmNetworker = require('megastore-swarm-networking') + var create = require('./helpers/create') test('basic read/write to/from a mount', t => { @@ -355,6 +360,72 @@ test('truncate within mount (with shared write capabilities)', async t => { }) }) +test('megastore mount replication between hyperdrives', async t => { + const megastore1 = new Megastore(path => raf('store1/' + path), memdb(), new SwarmNetworker()) + const megastore2 = new Megastore(path => raf('store2/' + path), memdb(), new SwarmNetworker()) + await megastore1.ready() + await megastore2.ready() + + megastore1.on('error', err => t.fail(err)) + megastore2.on('error', err => t.fail(err)) + + const cs1 = megastore1.get('cs1') + const cs2 = megastore1.get('cs2') + const cs3 = megastore2.get('cs3') + + const drive1 = create({ corestore: cs1 }) + const drive2 = create({ corestore: cs2 }) + var drive3 = null + + await new Promise(resolve => { + drive1.ready(err => { + t.error(err, 'no error') + drive3 = create(drive1.key, { corestore: cs3 }) + drive2.ready(err => { + t.error(err, 'no error') + drive3.ready(err => { + t.error(err, 'no error') + onready() + }) + }) + }) + + function onready() { + drive1.writeFile('hey', 'hi', err => { + t.error(err, 'no error') + drive2.writeFile('hello', 'world', err => { + t.error(err, 'no error') + drive1.mount('a', drive2.key, err => { + t.error(err, 'no error') + drive3.ready(err => { + return setTimeout(onmount, 100) + }) + }) + }) + }) + } + + function onmount () { + drive3.readFile('hey', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hi')) + drive3.readFile('a/hello', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('world')) + return resolve() + }) + }) + } + }) + + await megastore1.close() + await megastore2.close() + + await cleanup(['store1', 'store2']) + + t.end() +}) + test('versioned mount') test('watch will unwatch on umount') @@ -379,3 +450,12 @@ function replicateAll (drives, opts) { return streams } + +async function cleanup (dirs) { + return Promise.all(dirs.map(dir => new Promise((resolve, reject) => { + rimraf(dir, err => { + if (err) return reject(err) + return resolve() + }) + }))) +} From 9d86d94b70928c8d6014bd258f07277bb3a51f29 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sun, 23 Jun 2019 16:14:22 +0200 Subject: [PATCH 089/108] Remove debugging statements --- package.json | 6 ++-- test/mount.js | 99 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 3da5ec92..d6088c37 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "filesystem-constants": "^1.0.0", "hypercore-byte-stream": "^1.0.2", "hyperdrive-schemas": "^0.9.0", - "mountable-hypertrie": "^0.9.0", + "mountable-hypertrie": "^0.9.6", "mutexify": "^1.2.0", "pump": "^3.0.0", "pumpify": "^1.5.1", @@ -40,8 +40,8 @@ }, "devDependencies": { "fuzzbuzz": "^2.0.0", - "megastore": "^0.9.0", - "megastore-swarm-networking": "^1.0.1", + "megastore": "^0.9.7", + "megastore-swarm-networking": "^1.0.2", "memdb": "^1.3.1", "random-access-file": "^2.1.3", "random-access-memory": "^3.1.1", diff --git a/test/mount.js b/test/mount.js index 487d86c3..f736c209 100644 --- a/test/mount.js +++ b/test/mount.js @@ -8,6 +8,12 @@ const corestore = require('random-access-corestore') const Megastore = require('megastore') const SwarmNetworker = require('megastore-swarm-networking') +function createNetworker () { + return new SwarmNetworker({ + bootstrap: false + }) +} + var create = require('./helpers/create') test('basic read/write to/from a mount', t => { @@ -361,8 +367,8 @@ test('truncate within mount (with shared write capabilities)', async t => { }) test('megastore mount replication between hyperdrives', async t => { - const megastore1 = new Megastore(path => raf('store1/' + path), memdb(), new SwarmNetworker()) - const megastore2 = new Megastore(path => raf('store2/' + path), memdb(), new SwarmNetworker()) + const megastore1 = new Megastore(path => raf('store1/' + path), memdb(), createNetworker()) + const megastore2 = new Megastore(path => raf('store2/' + path), memdb(), createNetworker()) await megastore1.ready() await megastore2.ready() @@ -426,6 +432,95 @@ test('megastore mount replication between hyperdrives', async t => { t.end() }) +test('megastore mount replication between hyperdrives, multiple, nested mounts', async t => { + const megastore1 = new Megastore(path => ram('store1/' + path), memdb(), createNetworker()) + const megastore2 = new Megastore(path => ram('store2/' + path), memdb(), createNetworker()) + await megastore1.ready() + await megastore2.ready() + + megastore1.on('error', err => t.fail(err)) + megastore2.on('error', err => t.fail(err)) + + const [d1, d2] = await createMountee() + const drive = await createMounter(d1, d2) + await verify(drive) + + await megastore1.close() + await megastore2.close() + + // await cleanup(['store1', 'store2']) + + t.end() + + function createMountee () { + const cs1 = megastore1.get('cs1') + const cs2 = megastore1.get('cs2') + const cs3 = megastore1.get('cs3') + const drive1 = create({ corestore: cs1 }) + const drive2 = create({ corestore: cs2 }) + const drive3 = create({ corestore: cs3 }) + + return new Promise(resolve => { + drive2.ready(err => { + t.error(err, 'no error') + drive3.ready(err => { + t.error(err, 'no error') + return onready() + }) + }) + + function onready () { + drive1.mount('a', drive2.key, err => { + t.error(err, 'no error') + drive1.mount('b', drive3.key, err => { + t.error(err, 'no error') + return onmount() + }) + }) + } + + function onmount () { + drive1.writeFile('a/dog', 'hello', err => { + t.error(err, 'no error') + drive1.writeFile('b/cat', 'goodbye', err => { + t.error(err, 'no error') + return resolve([drive2, drive3]) + }) + }) + } + }) + } + + function createMounter (d2, d3) { + const cs1 = megastore2.get('cs1') + const drive1 = create({ corestore: cs1 }) + + return new Promise(resolve => { + drive1.mount('a', d2.key, err => { + t.error(err, 'no error') + drive1.mount('b', d3.key, err => { + t.error(err, 'no error') + return resolve(drive1) + }) + }) + }) + } + + function verify (drive) { + return new Promise(resolve => { + drive.readFile('a/dog', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + drive.readFile('b/cat', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('goodbye')) + return resolve() + }) + }) + }) + } +}) + test('versioned mount') test('watch will unwatch on umount') From 267198f4677a7d9feb57eecf3dc6c059c096f178 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Sun, 30 Jun 2019 23:31:45 +0200 Subject: [PATCH 090/108] Updated tests to remove megastore and reuse single corestores --- index.js | 24 ++++++++--- lib/storage.js | 7 +--- package.json | 3 +- test/mount.js | 110 +++++++++++++++++-------------------------------- 4 files changed, 60 insertions(+), 84 deletions(-) diff --git a/index.js b/index.js index 24fdea67..471e5582 100644 --- a/index.js +++ b/index.js @@ -9,6 +9,7 @@ const duplexify = require('duplexify') const through = require('through2') const pump = require('pump') +const { Corestore } = require('random-access-corestore') const coreByteStream = require('hypercore-byte-stream') const MountableHypertrie = require('mountable-hypertrie') @@ -42,14 +43,27 @@ class Hyperdrive extends EventEmitter { this.sparse = opts.sparse !== false this.sparseMetadata = opts.sparseMetadata !== false - this._corestore = defaultCorestore(storage, { - valueEncoding: 'binary' + this._corestore = defaultCorestore(storage, opts, { + valueEncoding: 'binary', + // TODO: Support mixed sparsity. + sparse: this.sparseMetadata || this.sparseMetadata }) - this.metadata = this._corestore.default({ + + const metadataOpts = { key, sparse: this.sparseMetadata, - secretKey: opts.secretKey, - }) + secretKey: (opts.keyPair) ? opts.keyPair.secretKey : opts.secretKey, + } + + if (storage instanceof Corestore && storage.isDefaultSet()){ + this.metadata = this._corestore.get({ + ...metadataOpts, + discoverable: true + }) + } else { + this.metadata = this._corestore.default(metadataOpts) + } + this._db = opts._db || new MountableHypertrie(this._corestore, key, { feed: this.metadata, sparse: this.sparseMetadata diff --git a/lib/storage.js b/lib/storage.js index 6573b0fa..0b052e36 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -1,8 +1,9 @@ const raf = require('random-access-file') const corestore = require('random-access-corestore') +const { Corestore } = corestore module.exports = function defaultCorestore (storage, opts) { - if (isCorestore(storage)) return storage + if ((storage instanceof Corestore)) return storage if (typeof storage === 'function') { var factory = path => storage(path) } else if (typeof storage === 'string') { @@ -10,7 +11,3 @@ module.exports = function defaultCorestore (storage, opts) { } return corestore(factory, opts) } - -function isCorestore (storage) { - return !!storage.get && !!storage.replicate && !!storage.close -} diff --git a/package.json b/package.json index d6088c37..85053a9f 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,7 @@ }, "devDependencies": { "fuzzbuzz": "^2.0.0", - "megastore": "^0.9.7", - "megastore-swarm-networking": "^1.0.2", + "corestore-swarm-networking": "^1.0.2", "memdb": "^1.3.1", "random-access-file": "^2.1.3", "random-access-memory": "^3.1.1", diff --git a/test/mount.js b/test/mount.js index f736c209..fba4927c 100644 --- a/test/mount.js +++ b/test/mount.js @@ -5,7 +5,6 @@ const memdb = require('memdb') const rimraf = require('rimraf') const corestore = require('random-access-corestore') -const Megastore = require('megastore') const SwarmNetworker = require('megastore-swarm-networking') function createNetworker () { @@ -217,16 +216,11 @@ test('cross-mount symlink', t => { }) test('lists nested mounts, shared write capabilities', async t => { - const megastore = new Megastore(ram, memdb(), false) - await megastore.ready() - - const cs1 = megastore.get('cs1') - const cs2 = megastore.get('cs2') - const cs3 = megastore.get('cs3') + const store = corestore(ram) - const drive1 = create({ corestore: cs1 }) - const drive2 = create({ corestore: cs2 }) - const drive3 = create({ corestore: cs3 }) + const drive1 = create({ corestore: store }) + const drive2 = create({ corestore: store }) + const drive3 = create({ corestore: store }) drive3.ready(err => { t.error(err, 'no error') @@ -276,14 +270,10 @@ test('independent corestores do not share write capabilities', t => { }) test('shared corestores will share write capabilities', async t => { - const megastore = new Megastore(ram, memdb(), false) - await megastore.ready() - - const cs1 = megastore.get('cs1') - const cs2 = megastore.get('cs2') + const store = corestore(ram) - const drive1 = create({ corestore: cs1 }) - const drive2 = create({ corestore: cs2 }) + const drive1 = create({ corestore: store }) + const drive2 = create({ corestore: store }) drive2.ready(err => { t.error(err, 'no error') @@ -334,14 +324,10 @@ test('can mount hypercores', async t => { }) test('truncate within mount (with shared write capabilities)', async t => { - const megastore = new Megastore(ram, memdb(), false) - await megastore.ready() - - const cs1 = megastore.get('cs1') - const cs2 = megastore.get('cs2') + const store = corestore(ram) - const drive1 = create({ corestore: cs1 }) - const drive2 = create({ corestore: cs2 }) + const drive1 = create({ corestore: store }) + const drive2 = create({ corestore: store }) drive2.ready(err => { t.error(err, 'no error') @@ -366,31 +352,24 @@ test('truncate within mount (with shared write capabilities)', async t => { }) }) -test('megastore mount replication between hyperdrives', async t => { - const megastore1 = new Megastore(path => raf('store1/' + path), memdb(), createNetworker()) - const megastore2 = new Megastore(path => raf('store2/' + path), memdb(), createNetworker()) - await megastore1.ready() - await megastore2.ready() - - megastore1.on('error', err => t.fail(err)) - megastore2.on('error', err => t.fail(err)) - - const cs1 = megastore1.get('cs1') - const cs2 = megastore1.get('cs2') - const cs3 = megastore2.get('cs3') +test('mount replication between hyperdrives', async t => { + const store1 = corestore(path => ram('cs1/' + path)) + const store2 = corestore(path => ram('cs2/' + path)) + const store3 = corestore(path => ram('cs3/' + path)) - const drive1 = create({ corestore: cs1 }) - const drive2 = create({ corestore: cs2 }) + const drive1 = create({ corestore: store1 }) + const drive2 = create({ corestore: store2 }) var drive3 = null await new Promise(resolve => { drive1.ready(err => { t.error(err, 'no error') - drive3 = create(drive1.key, { corestore: cs3 }) + drive3 = create(drive1.key, { corestore: store3 }) drive2.ready(err => { t.error(err, 'no error') drive3.ready(err => { t.error(err, 'no error') + replicateAll([drive1, drive2, drive3]) onready() }) }) @@ -424,48 +403,32 @@ test('megastore mount replication between hyperdrives', async t => { } }) - await megastore1.close() - await megastore2.close() - - await cleanup(['store1', 'store2']) - t.end() }) -test('megastore mount replication between hyperdrives, multiple, nested mounts', async t => { - const megastore1 = new Megastore(path => ram('store1/' + path), memdb(), createNetworker()) - const megastore2 = new Megastore(path => ram('store2/' + path), memdb(), createNetworker()) - await megastore1.ready() - await megastore2.ready() - - megastore1.on('error', err => t.fail(err)) - megastore2.on('error', err => t.fail(err)) - +test('mount replication between hyperdrives, multiple, nested mounts', async t => { const [d1, d2] = await createMountee() const drive = await createMounter(d1, d2) await verify(drive) - await megastore1.close() - await megastore2.close() - - // await cleanup(['store1', 'store2']) - t.end() function createMountee () { - const cs1 = megastore1.get('cs1') - const cs2 = megastore1.get('cs2') - const cs3 = megastore1.get('cs3') - const drive1 = create({ corestore: cs1 }) - const drive2 = create({ corestore: cs2 }) - const drive3 = create({ corestore: cs3 }) + const store = corestore(path => ram('cs1/' + path)) + const drive1 = create({ corestore: store }) + var drive2, drive3 return new Promise(resolve => { - drive2.ready(err => { + drive1.ready(err => { t.error(err, 'no error') - drive3.ready(err => { + drive2 = create({ corestore: store }) + drive3 = create({ corestore: store }) + drive2.ready(err => { t.error(err, 'no error') - return onready() + drive3.ready(err => { + t.error(err, 'no error') + return onready() + }) }) }) @@ -492,15 +455,18 @@ test('megastore mount replication between hyperdrives, multiple, nested mounts', } function createMounter (d2, d3) { - const cs1 = megastore2.get('cs1') - const drive1 = create({ corestore: cs1 }) + const drive1 = create({ corestore: corestore(path => ram('cs4/' + path)) }) return new Promise(resolve => { - drive1.mount('a', d2.key, err => { + drive1.ready(err => { t.error(err, 'no error') - drive1.mount('b', d3.key, err => { + replicateAll([drive1, d2, d3]) + drive1.mount('a', d2.key, err => { t.error(err, 'no error') - return resolve(drive1) + drive1.mount('b', d3.key, err => { + t.error(err, 'no error') + setTimeout(() => resolve(drive1), 1000) + }) }) }) }) From 30f38d4c4947b6926dd18ecb81bc84b096501529 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 8 Jul 2019 11:20:16 +0200 Subject: [PATCH 091/108] Create new statOpts inside mount --- index.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index 471e5582..4f7e2d13 100644 --- a/index.js +++ b/index.js @@ -282,7 +282,7 @@ class Hyperdrive extends EventEmitter { this.ready(err => { if (err) return stream.destroy(err) - this.stat(name, (err, st, trie) => { + this.stat(name, { file: true }, (err, st, trie) => { if (err) return stream.destroy(err) this._getContent(trie, (err, contentState) => { if (err) return stream.destroy(err) @@ -723,14 +723,15 @@ class Hyperdrive extends EventEmitter { path = fixName(path) opts = opts || {} + const statOpts = {} - opts.mount = { + statOpts.mount = { key, version: opts.version, hash: opts.hash, hypercore: !!opts.hypercore } - opts.directory = !opts.hypercore + statOpts.directory = !opts.hypercore if (opts.hypercore) { const core = this._corestore.get({ @@ -740,8 +741,8 @@ class Hyperdrive extends EventEmitter { }) core.ready(err => { if (err) return cb(err) - opts.size = core.byteLength - opts.blocks = core.length + statOpts.size = core.byteLength + statOpts.blocks = core.length return mountCore() }) } else { @@ -749,14 +750,14 @@ class Hyperdrive extends EventEmitter { } function mountCore () { - self._createStat(path, opts, (err, st) => { + self._createStat(path, statOpts, (err, st) => { if (err) return cb(err) return self._db.put(path, st.encode(), cb) }) } function mountTrie () { - self._createStat(path, opts, (err, st) => { + self._createStat(path, statOpts, (err, st) => { if (err) return cb(err) self._db.mount(path, key, { ...opts, value: st.encode() }, err => { if (err) return cb(err) From 993a16f4613b0923ebdf729e66445a37cded425a Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 8 Jul 2019 11:30:33 +0200 Subject: [PATCH 092/108] Remove unused deps --- package.json | 4 +--- test/mount.js | 19 ------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/package.json b/package.json index 85053a9f..54c5ae3e 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "hyperdrive-schemas": "^0.9.0", "mountable-hypertrie": "^0.9.6", "mutexify": "^1.2.0", + "nanoiterator": "^1.2.0", "pump": "^3.0.0", "pumpify": "^1.5.1", "random-access-corestore": "^0.9.1", @@ -40,11 +41,8 @@ }, "devDependencies": { "fuzzbuzz": "^2.0.0", - "corestore-swarm-networking": "^1.0.2", - "memdb": "^1.3.1", "random-access-file": "^2.1.3", "random-access-memory": "^3.1.1", - "rimraf": "^2.6.3", "tape": "^4.10.0", "temporary-directory": "^1.0.2" } diff --git a/test/mount.js b/test/mount.js index fba4927c..332c77e7 100644 --- a/test/mount.js +++ b/test/mount.js @@ -1,18 +1,8 @@ var test = require('tape') const ram = require('random-access-memory') const raf = require('random-access-file') -const memdb = require('memdb') -const rimraf = require('rimraf') const corestore = require('random-access-corestore') -const SwarmNetworker = require('megastore-swarm-networking') - -function createNetworker () { - return new SwarmNetworker({ - bootstrap: false - }) -} - var create = require('./helpers/create') test('basic read/write to/from a mount', t => { @@ -511,12 +501,3 @@ function replicateAll (drives, opts) { return streams } - -async function cleanup (dirs) { - return Promise.all(dirs.map(dir => new Promise((resolve, reject) => { - rimraf(dir, err => { - if (err) return reject(err) - return resolve() - }) - }))) -} From 233078219e195766d798b2708f8067d1a82eb5e6 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 8 Jul 2019 12:08:12 +0200 Subject: [PATCH 093/108] Update corestore dep --- index.js | 2 +- lib/storage.js | 2 +- package.json | 2 +- test/mount.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 4f7e2d13..79b18050 100644 --- a/index.js +++ b/index.js @@ -9,7 +9,7 @@ const duplexify = require('duplexify') const through = require('through2') const pump = require('pump') -const { Corestore } = require('random-access-corestore') +const { Corestore } = require('corestore') const coreByteStream = require('hypercore-byte-stream') const MountableHypertrie = require('mountable-hypertrie') diff --git a/lib/storage.js b/lib/storage.js index 0b052e36..e453a297 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -1,5 +1,5 @@ const raf = require('random-access-file') -const corestore = require('random-access-corestore') +const corestore = require('corestore') const { Corestore } = corestore module.exports = function defaultCorestore (storage, opts) { diff --git a/package.json b/package.json index 54c5ae3e..d803e505 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "nanoiterator": "^1.2.0", "pump": "^3.0.0", "pumpify": "^1.5.1", - "random-access-corestore": "^0.9.1", + "corestore": "^2.0.0", "sodium-universal": "^2.0.0", "stream-collector": "^1.0.1", "through2": "^3.0.0", diff --git a/test/mount.js b/test/mount.js index 332c77e7..8a8d7a9d 100644 --- a/test/mount.js +++ b/test/mount.js @@ -2,7 +2,7 @@ var test = require('tape') const ram = require('random-access-memory') const raf = require('random-access-file') -const corestore = require('random-access-corestore') +const corestore = require('corestore') var create = require('./helpers/create') test('basic read/write to/from a mount', t => { From c277459e0aae46f951a0352b4c7ba828565c6d9c Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Mon, 8 Jul 2019 13:49:00 +0200 Subject: [PATCH 094/108] Pass uid/gid into mount opts --- index.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 79b18050..87cef877 100644 --- a/index.js +++ b/index.js @@ -723,8 +723,11 @@ class Hyperdrive extends EventEmitter { path = fixName(path) opts = opts || {} - const statOpts = {} + const statOpts = { + uid: opts.uid, + gid: opts.gid + } statOpts.mount = { key, version: opts.version, From 385bf7b138f4acf31b0603aa642abf4b10e890a4 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 9 Jul 2019 13:07:13 +0200 Subject: [PATCH 095/108] 10.0.0-rc5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d803e505..00c737e3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc4", + "version": "10.0.0-rc5", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From 5262c7df45c1bbc9faf200327f47de3016a48b35 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 9 Jul 2019 15:34:18 +0200 Subject: [PATCH 096/108] Pass flags in _update + add unmount and unmount test --- index.js | 14 +++++++++++++- test/mount.js | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 87cef877..53535741 100644 --- a/index.js +++ b/index.js @@ -228,7 +228,7 @@ class Hyperdrive extends EventEmitter { return cb(err) } const newStat = Object.assign(decoded, stat) - return this._putStat(name, newStat, cb) + return this._putStat(name, newStat, { flags: st.flags }, cb) }) } @@ -770,6 +770,18 @@ class Hyperdrive extends EventEmitter { } } + unmount (path, cb) { + this.stat(path, (err, st) => { + if (err) return cb(err) + if (!st.mount) return cb(new Error('Can only unmount mounts.')) + if (st.mount.hypercore) { + return this.unlink(path, cb) + } else { + return this._db.unmount(path, cb) + } + }) + } + symlink (target, linkName, cb) { target = unixify(target) linkName = fixName(linkName) diff --git a/test/mount.js b/test/mount.js index 8a8d7a9d..8206e026 100644 --- a/test/mount.js +++ b/test/mount.js @@ -14,7 +14,8 @@ test('basic read/write to/from a mount', t => { drive2.ready(err => { t.error(err, 'no error') - drive2.writeFile('b', 'hello', err => {t.error(err, 'no error') + drive2.writeFile('b', 'hello', err => { + t.error(err, 'no error') drive1.mount('a', drive2.key, err => { t.error(err, 'no error') drive1.readFile('a/b', (err, contents) => { @@ -27,6 +28,40 @@ test('basic read/write to/from a mount', t => { }) }) +test('can delete a mount', t => { + const drive1 = create() + const drive2 = create() + + const s1 = drive1.replicate({ live: true, encrypt: false }) + s1.pipe(drive2.replicate({ live: true, encrypt: false })).pipe(s1) + + drive2.ready(err => { + t.error(err, 'no error') + drive2.writeFile('b', 'hello', err => { + t.error(err, 'no error') + drive1.mount('a', drive2.key, err => { + t.error(err, 'no error') + drive1.readFile('a/b', (err, contents) => { + t.error(err, 'no error') + t.same(contents, Buffer.from('hello')) + return deleteMount() + }) + }) + }) + }) + + function deleteMount () { + drive1.unmount('a', err => { + t.error(err, 'no error') + drive1.readFile('a/b', (err, contents) => { + t.true(err) + t.same(err.errno, 2) + t.end() + }) + }) + } +}) + test('multiple flat mounts', t => { const drive1 = create() const drive2 = create() From ff8a96c1f7e36a7df99b136b7d0cba1592dc1255 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 9 Jul 2019 15:34:34 +0200 Subject: [PATCH 097/108] 10.0.0-rc6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 00c737e3..d1ef9dc8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc5", + "version": "10.0.0-rc6", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From 065a5407d3b1922e576c0ec2110d1b5563a1b65c Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 9 Jul 2019 16:01:27 +0200 Subject: [PATCH 098/108] Bump mountable-hypertrie dep --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d1ef9dc8..a9b9eeb8 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "filesystem-constants": "^1.0.0", "hypercore-byte-stream": "^1.0.2", "hyperdrive-schemas": "^0.9.0", - "mountable-hypertrie": "^0.9.6", + "mountable-hypertrie": "^0.10.0", "mutexify": "^1.2.0", "nanoiterator": "^1.2.0", "pump": "^3.0.0", From e47b82a6ad6aa3493e33abe34d6abb64597af1c8 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 9 Jul 2019 16:02:05 +0200 Subject: [PATCH 099/108] 10.0.0-rc7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a9b9eeb8..e2dfeeb4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc6", + "version": "10.0.0-rc7", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From b848e784ab9c2c65b028d6d793ab057f0f6cdb85 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 11 Jul 2019 16:35:02 +0200 Subject: [PATCH 100/108] Add listAllMounts --- index.js | 37 +++++++++----- lib/fd.js | 3 +- lib/iterator.js | 54 +++++++++++++++++++-- test/basic.js | 6 +-- test/checkout.js | 1 - test/deletion.js | 1 - test/diff.js | 6 +-- test/fuzzing.js | 6 +-- test/mount.js | 123 ++++++++++++++++++++++++++++++++++++++++++++--- test/readdir.js | 5 +- test/stat.js | 2 +- test/storage.js | 4 +- test/symlink.js | 2 +- test/watch.js | 1 - 14 files changed, 207 insertions(+), 44 deletions(-) diff --git a/index.js b/index.js index 53535741..02309957 100644 --- a/index.js +++ b/index.js @@ -4,7 +4,6 @@ const { EventEmitter } = require('events') const collect = require('stream-collector') const thunky = require('thunky') const unixify = require('unixify') -const mutexify = require('mutexify') const duplexify = require('duplexify') const through = require('through2') const pump = require('pump') @@ -18,7 +17,7 @@ const Stat = require('./lib/stat') const errors = require('./lib/errors') const defaultCorestore = require('./lib/storage') const { contentKeyPair, contentOptions, ContentState } = require('./lib/content') -const { createStatStream } = require('./lib/iterator') +const { createStatStream, createMountStream } = require('./lib/iterator') // 20 is arbitrary, just to make the fds > stdio etc const STDIO_CAP = 20 @@ -43,19 +42,20 @@ class Hyperdrive extends EventEmitter { this.sparse = opts.sparse !== false this.sparseMetadata = opts.sparseMetadata !== false - this._corestore = defaultCorestore(storage, opts, { + this._corestore = defaultCorestore(storage, { + ...opts, valueEncoding: 'binary', // TODO: Support mixed sparsity. - sparse: this.sparseMetadata || this.sparseMetadata + sparse: this.sparse || this.sparseMetadata }) const metadataOpts = { key, sparse: this.sparseMetadata, - secretKey: (opts.keyPair) ? opts.keyPair.secretKey : opts.secretKey, + secretKey: (opts.keyPair) ? opts.keyPair.secretKey : opts.secretKey } - if (storage instanceof Corestore && storage.isDefaultSet()){ + if (storage instanceof Corestore && storage.isDefaultSet()) { this.metadata = this._corestore.get({ ...metadataOpts, discoverable: true @@ -443,7 +443,7 @@ class Hyperdrive extends EventEmitter { readFile (name, opts, cb) { if (typeof opts === 'function') return this.readFile(name, null, opts) - if (typeof opts === 'string') opts = {encoding: opts} + if (typeof opts === 'string') opts = { encoding: opts } if (!opts) opts = {} name = fixName(name) @@ -530,7 +530,7 @@ class Hyperdrive extends EventEmitter { mkdir (name, opts, cb) { if (typeof opts === 'function') return this.mkdir(name, null, opts) - if (typeof opts === 'number') opts = {mode: opts} + if (typeof opts === 'number') opts = { mode: opts } if (!opts) opts = {} if (!cb) cb = noop @@ -701,10 +701,6 @@ class Hyperdrive extends EventEmitter { if (typeof fd === 'number') return this._closeFile(fd, cb || noop) else cb = fd if (!cb) cb = noop - const self = this - - // Attempt to close all feeds, even if a subset of them fail. Return the last error. - var closeErr = null this.ready(err => { if (err) return cb(err) @@ -795,6 +791,23 @@ class Hyperdrive extends EventEmitter { return this._putStat(linkName, st, cb) }) } + + createMountStream (opts) { + return createMountStream(this, this._db, opts) + } + + getAllMounts (opts, cb) { + if (typeof opts === 'function') return this.getAllMounts(null, opts) + const mounts = new Map() + + collect(this.createMountStream(opts), (err, mountList) => { + if (err) return cb(err) + for (const { path, metadata, content } of mountList) { + mounts.set(path, { metadata, content }) + } + return cb(null, mounts) + }) + } } function isObject (val) { diff --git a/lib/fd.js b/lib/fd.js index 68ad19a5..5b053123 100644 --- a/lib/fd.js +++ b/lib/fd.js @@ -2,7 +2,6 @@ const byteStream = require('byte-stream') const through = require('through2') const pumpify = require('pumpify') -const { messages } = require('hyperdrive-schemas') const errors = require('./errors') const { linux: linuxConstants, parse } = require('filesystem-constants') @@ -15,7 +14,7 @@ const { O_APPEND, O_SYNC, O_EXCL, - O_ACCMODE, + O_ACCMODE } = linuxConstants class FileDescriptor { diff --git a/lib/iterator.js b/lib/iterator.js index 38ad0ca0..f89977b1 100644 --- a/lib/iterator.js +++ b/lib/iterator.js @@ -52,24 +52,69 @@ function statIterator (drive, db, path, opts) { function pushLink (nodePath, linkPath, cb) { drive.stat(linkPath, (err, stat, _, resolvedLink) => { + if (err) return cb(err) if (!stat) return cb(null) if (stat.isDirectory()) { - if (opts && opts.recursive || nodePath === path) { + if ((opts && opts.recursive) || (nodePath === path)) { stack.unshift({ path: nodePath, target: resolvedLink, iterator: db.iterator(resolvedLink, { gt: true, ...opts }) }) return cb(null) } - return cb(null, { stat, path: linkPath}) + return cb(null, { stat, path: linkPath }) } return cb(null, stat) }) } } +function mountIterator (drive, db, opts) { + var ite = db.mountIterator(opts) + var first = drive + + return nanoiterator({ + next: function next (cb) { + if (first) { + first = null + return cb(null, { + path: '/', + metadata: drive.metadata, + content: drive._contentStates.get(db).feed + }) + } + + ite.next((err, val) => { + if (err) return cb(err) + if (!val) return cb(null, null) + + const contentState = drive._contentStates.get(val.trie) + if (contentState) return process.nextTick(oncontent, val.path, val.trie, contentState) + return drive._getContent(val.trie, (err, contentState) => { + if (err) return cb(err) + return oncontent(val.path, val.trie, contentState) + }) + }) + + function oncontent (path, trie, contentState) { + return cb(null, { + path, + // TODO: this means the feed property should not be hidden + metadata: trie._trie.feed, + content: contentState.feed + }) + } + } + }) +} + function createStatStream (drive, db, path, opts) { const ite = statIterator(drive, db, path, opts) return toStream(ite) } +function createMountStream (drive, db, opts) { + const ite = mountIterator(drive, db, opts) + return toStream(ite) +} + function prefix (key) { if (key.startsWith('/')) return key return '/' + key @@ -77,6 +122,7 @@ function prefix (key) { module.exports = { statIterator, - createStatStream + createStatStream, + mountIterator, + createMountStream } - diff --git a/test/basic.js b/test/basic.js index 0f5a3f5d..5702e529 100644 --- a/test/basic.js +++ b/test/basic.js @@ -55,10 +55,10 @@ tape('write and read (sparse)', function (t) { var drive = create() drive.on('ready', function () { - var clone = create(drive.key, {sparse: true}) + var clone = create(drive.key, { sparse: true }) var s1 = clone.replicate({ live: true, encrypt: false }) - var s2 = drive.replicate({ live: true, encrypt: false}) + var s2 = drive.replicate({ live: true, encrypt: false }) s1.pipe(s2).pipe(s1) drive.writeFile('/hello.txt', 'world', function (err) { @@ -92,7 +92,7 @@ tape('provide keypair', function (t) { sodium.crypto_sign_keypair(publicKey, secretKey) - var drive = create(publicKey, {secretKey: secretKey}) + var drive = create(publicKey, { secretKey: secretKey }) drive.on('ready', function () { t.ok(drive.writable) diff --git a/test/checkout.js b/test/checkout.js index b51c5574..29a3437e 100644 --- a/test/checkout.js +++ b/test/checkout.js @@ -86,4 +86,3 @@ tape.skip('closing a read-only, latest clone', function (t) { t.end() }) }) - diff --git a/test/deletion.js b/test/deletion.js index 7ffc3a5a..473cf18c 100644 --- a/test/deletion.js +++ b/test/deletion.js @@ -15,4 +15,3 @@ tape('write and unlink', function (t) { }) }) }) - diff --git a/test/diff.js b/test/diff.js index 1cc5e3af..19ee4954 100644 --- a/test/diff.js +++ b/test/diff.js @@ -6,13 +6,13 @@ tape('simple diff stream', async function (t) { var v1, v2, v3 let v3Diff = ['del-hello'] - let v2Diff = [...v3Diff, 'put-other'] + let v2Diff = [...v3Diff, 'put-other'] let v1Diff = [...v2Diff, 'put-hello'] await writeVersions() console.log('drive.version:', drive.version, 'v1:', v1) - // await verifyDiffStream(v1, v1Diff) - // await verifyDiffStream(v2, v2Diff) + await verifyDiffStream(v1, v1Diff) + await verifyDiffStream(v2, v2Diff) await verifyDiffStream(v3, v3Diff) t.end() diff --git a/test/fuzzing.js b/test/fuzzing.js index c4b68890..341bbdc3 100644 --- a/test/fuzzing.js +++ b/test/fuzzing.js @@ -21,12 +21,12 @@ class HyperdriveFuzzer extends FuzzBuzz { this.add(5, this.randomStatefulFileDescriptorRead) this.add(5, this.randomStatefulFileDescriptorWrite) this.add(3, this.statFile) - //this.add(3, this.statDirectory) + // this.add(3, this.statDirectory) this.add(2, this.deleteInvalidFile) this.add(2, this.randomReadStream) this.add(2, this.randomStatelessFileDescriptorRead) this.add(1, this.createReadableFileDescriptor) - //this.add(1, this.writeAndMkdir) + // this.add(1, this.writeAndMkdir) } // START Helper functions. @@ -250,7 +250,7 @@ class HyperdriveFuzzer extends FuzzBuzz { collect(stream, (err, bufs) => { if (err) return reject(err) let buf = bufs.length === 1 ? bufs[0] : Buffer.concat(bufs) - + if (!buf.equals(content.slice(start, start + length))) { console.log('buf:', buf, 'content slice:', content.slice(start, start + length)) return reject(new Error('Read stream does not match content slice.')) diff --git a/test/mount.js b/test/mount.js index 8206e026..d5f11a3a 100644 --- a/test/mount.js +++ b/test/mount.js @@ -1,6 +1,5 @@ var test = require('tape') const ram = require('random-access-memory') -const raf = require('random-access-file') const corestore = require('corestore') var create = require('./helpers/create') @@ -72,7 +71,9 @@ test('multiple flat mounts', t => { replicateAll([drive1, drive2, drive3]) drive3.ready(err => { + t.error(err, 'no error') drive2.ready(err => { + t.error(err, 'no error') key1 = drive2.key key2 = drive3.key onready() @@ -121,7 +122,9 @@ test('recursive mounts', async t => { replicateAll([drive1, drive2, drive3]) drive3.ready(err => { + t.error(err, 'no error') drive2.ready(err => { + t.error(err, 'no error') key1 = drive2.key key2 = drive3.key onready() @@ -260,6 +263,7 @@ test('lists nested mounts, shared write capabilities', async t => { function onmount () { drive2.lstat('b', (err, stat) => { + t.error(err, 'no error') drive1.readdir('a', (err, list) => { t.error(err, 'no error') t.same(list, ['b']) @@ -269,9 +273,6 @@ test('lists nested mounts, shared write capabilities', async t => { } }) -test('dynamically resolves cross-mount symlinks') -test('symlinks cannot break the sandbox') - test('independent corestores do not share write capabilities', t => { const drive1 = create() const drive2 = create() @@ -400,7 +401,7 @@ test('mount replication between hyperdrives', async t => { }) }) - function onready() { + function onready () { drive1.writeFile('hey', 'hi', err => { t.error(err, 'no error') drive2.writeFile('hello', 'world', err => { @@ -408,6 +409,7 @@ test('mount replication between hyperdrives', async t => { drive1.mount('a', drive2.key, err => { t.error(err, 'no error') drive3.ready(err => { + t.error(err, 'no error') return setTimeout(onmount, 100) }) }) @@ -512,6 +514,115 @@ test('mount replication between hyperdrives, multiple, nested mounts', async t = } }) +test('can list in-memory mounts', async t => { + const drive1 = create() + const drive2 = create() + const drive3 = create() + + var key1, key2 + + replicateAll([drive1, drive2, drive3]) + + + drive3.ready(err => { + t.error(err, 'no error') + drive2.ready(err => { + t.error(err, 'no error') + key1 = drive2.key + key2 = drive3.key + onready() + }) + }) + + function onready () { + drive2.writeFile('a', 'hello', err => { + t.error(err, 'no error') + drive3.writeFile('b', 'world', err => { + t.error(err, 'no error') + onwrite() + }) + }) + } + + function onwrite () { + drive1.mount('a', key1, err => { + t.error(err, 'no error') + drive1.mount('b', key2, err => { + t.error(err, 'no error') + onmount() + }) + }) + } + + function onmount () { + drive1.readFile('a/a', (err, contents) => { + t.error(err, 'no error') + t.true(contents) + drive1.getAllMounts({ memory: true }, (err, mounts) => { + t.error(err, 'no error') + t.same(mounts.size, 2) + t.true(mounts.get('/')) + t.true(mounts.get('/a')) + t.end() + }) + }) + } +}) + +test('can list all mounts (including those not in memory)', async t => { + const drive1 = create() + const drive2 = create() + const drive3 = create() + + var key1, key2 + + replicateAll([drive1, drive2, drive3]) + + drive3.ready(err => { + t.error(err, 'no error') + drive2.ready(err => { + t.error(err, 'no error') + key1 = drive2.key + key2 = drive3.key + onready() + }) + }) + + function onready () { + drive2.writeFile('a', 'hello', err => { + t.error(err, 'no error') + drive3.writeFile('b', 'world', err => { + t.error(err, 'no error') + onwrite() + }) + }) + } + + function onwrite () { + drive1.mount('a', key1, err => { + t.error(err, 'no error') + drive1.mount('b', key2, err => { + t.error(err, 'no error') + onmount() + }) + }) + } + + function onmount () { + drive1.getAllMounts((err, mounts) => { + t.error(err, 'no error') + t.same(mounts.size, 3) + t.true(mounts.get('/')) + t.true(mounts.get('/a')) + t.true(mounts.get('/b')) + t.end() + }) + } +}) + +test('can list in-memory mounts recursively') +test('dynamically resolves cross-mount symlinks') +test('symlinks cannot break the sandbox') test('versioned mount') test('watch will unwatch on umount') @@ -525,7 +636,7 @@ function replicateAll (drives, opts) { const dest = drives[j] if (i === j || replicated.has(j)) continue - const s1 = source.replicate({ ...opts, live: true, encrypt: false}) + const s1 = source.replicate({ ...opts, live: true, encrypt: false }) const s2 = dest.replicate({ ...opts, live: true, encrypt: false }) streams.push([s1, s2]) diff --git a/test/readdir.js b/test/readdir.js index 215ccc3f..f626b49e 100644 --- a/test/readdir.js +++ b/test/readdir.js @@ -1,6 +1,5 @@ const crypto = require('crypto') const test = require('tape') -const collect = require('stream-collector') const create = require('./helpers/create') const { runAll } = require('./helpers/util') @@ -25,7 +24,7 @@ test('readdir on empty directory', async function (t) { cb => drive.mkdir('l', cb), cb => writeFiles(drive, files, cb), cb => validateReaddir(t, drive, 'd', [], cb), - cb => validateReaddir(t, drive, 'l' ,[], cb) + cb => validateReaddir(t, drive, 'l', [], cb) ]) } catch (err) { t.fail(err) @@ -314,8 +313,6 @@ test('can stream a large directory', async function (t) { } }) - - function validateReaddir (t, drive, path, names, opts, cb) { if (typeof opts === 'function') return validateReaddir(t, drive, path, names, {}, opts) drive.readdir(path, opts, (err, list) => { diff --git a/test/stat.js b/test/stat.js index 19e15817..927954bb 100644 --- a/test/stat.js +++ b/test/stat.js @@ -6,7 +6,7 @@ var mask = 511 // 0b111111111 tape('stat file', function (t) { var drive = create() - drive.writeFile('/foo', 'bar', {mode: 438}, function (err) { + drive.writeFile('/foo', 'bar', { mode: 438 }, function (err) { t.error(err, 'no error') drive.stat('/foo', function (err, st) { t.error(err, 'no error') diff --git a/test/storage.js b/test/storage.js index adebbac9..712af4be 100644 --- a/test/storage.js +++ b/test/storage.js @@ -79,7 +79,7 @@ tape('write and read (sparse)', function (t) { t.ifError(err) var drive = hyperdrive(dir) drive.on('ready', function () { - var clone = create(drive.key, {sparse: true}) + var clone = create(drive.key, { sparse: true }) clone.on('ready', function () { drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') @@ -103,7 +103,7 @@ tape('write and read (sparse)', function (t) { tape('sparse read/write two files', function (t) { var drive = create() drive.on('ready', function () { - var clone = create(drive.key, {sparse: true}) + var clone = create(drive.key, { sparse: true }) drive.writeFile('/hello.txt', 'world', function (err) { t.error(err, 'no error') drive.writeFile('/hello2.txt', 'world', function (err) { diff --git a/test/symlink.js b/test/symlink.js index 2aa66e1d..27f4e09a 100644 --- a/test/symlink.js +++ b/test/symlink.js @@ -90,7 +90,7 @@ test('symlinks appear in readdir', t => { }) }) - function onlink () { + function onlink () { archive.readdir('/', (err, files) => { t.error(err, 'no errors') t.same(files, ['hello.txt', 'link.txt']) diff --git a/test/watch.js b/test/watch.js index 38883b81..47c15b5a 100644 --- a/test/watch.js +++ b/test/watch.js @@ -27,4 +27,3 @@ tape('simple watch', function (t) { }) } }) - From 3261a0b1a7cf675bb5c8240eb2cb6e582b1a0603 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 11 Jul 2019 16:38:37 +0200 Subject: [PATCH 101/108] 10.0.0-rc8 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e2dfeeb4..178e7391 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc7", + "version": "10.0.0-rc8", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From f4356eb95f6de7448a28675ac092a3638e1fec70 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 18 Jul 2019 22:26:27 +0200 Subject: [PATCH 102/108] Added broken symlink readdir test + getAllMounts test --- lib/iterator.js | 25 ++++++++++++------------- lib/storage.js | 2 +- test/mount.js | 13 +++++++++++++ test/readdir.js | 15 +++++---------- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/lib/iterator.js b/lib/iterator.js index f89977b1..e8475065 100644 --- a/lib/iterator.js +++ b/lib/iterator.js @@ -38,9 +38,9 @@ function statIterator (drive, db, path, opts) { } else { linkPath = p.resolve('/', p.dirname(node.key), st.linkname) } - return pushLink(prefix(node.key), linkPath, (err, linkStat) => { + return pushLink(prefix(node.key), linkPath, st, (err, linkStat) => { if (err) return cb(err) - if (linkStat) return cb(null, { stat: linkStat, path: prefix(node.key) }) + if (linkStat) return cb(null, { stat: st, path: prefix(node.key) }) return next(cb) }) } @@ -50,19 +50,18 @@ function statIterator (drive, db, path, opts) { }) } - function pushLink (nodePath, linkPath, cb) { - drive.stat(linkPath, (err, stat, _, resolvedLink) => { - if (err) return cb(err) - if (!stat) return cb(null) - if (stat.isDirectory()) { - if ((opts && opts.recursive) || (nodePath === path)) { + function pushLink (nodePath, linkPath, stat, cb) { + if (opts && opts.recursive || (nodePath === path)) { + return drive.stat(linkPath, (err, targetStat, _, resolvedLink) => { + if (err && err.errno !== 2) return cb(err) + if (!targetStat) return cb(null) + if (targetStat.isDirectory()) { stack.unshift({ path: nodePath, target: resolvedLink, iterator: db.iterator(resolvedLink, { gt: true, ...opts }) }) - return cb(null) } - return cb(null, { stat, path: linkPath }) - } - return cb(null, stat) - }) + return cb(null, stat) + }) + } + return process.nextTick(cb, null, stat) } } diff --git a/lib/storage.js b/lib/storage.js index e453a297..0cbbc491 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -3,7 +3,7 @@ const corestore = require('corestore') const { Corestore } = corestore module.exports = function defaultCorestore (storage, opts) { - if ((storage instanceof Corestore)) return storage + if (storage instanceof Corestore) return storage if (typeof storage === 'function') { var factory = path => storage(path) } else if (typeof storage === 'string') { diff --git a/test/mount.js b/test/mount.js index d5f11a3a..5c09f340 100644 --- a/test/mount.js +++ b/test/mount.js @@ -569,6 +569,19 @@ test('can list in-memory mounts', async t => { } }) +test('getAllMounts with no mounts returns only the root mount', async t => { + const drive1 = create() + drive1.ready(err => { + t.error(err, 'no error') + drive1.getAllMounts({ memory: true}, (err, mounts) => { + t.error(err, 'no error') + t.true(mounts) + t.same(mounts.size, 1) + t.end() + }) + }) +}) + test('can list all mounts (including those not in memory)', async t => { const drive1 = create() const drive2 = create() diff --git a/test/readdir.js b/test/readdir.js index f626b49e..be8ef4dd 100644 --- a/test/readdir.js +++ b/test/readdir.js @@ -159,7 +159,7 @@ test('readdir follows symlink', async t => { t.end() }) -test('readdir follows symlink', async t => { +test('readdir works with broken links', async t => { const drive = create() const files = createFiles([ @@ -175,22 +175,17 @@ test('readdir follows symlink', async t => { ]) const links = new Map([ ['f', 'a'], - ['p', 'a/c'], + ['p', 'nothing_here'], ['g', 'e'] ]) - const fExpected = ['f/b', 'f/c/d', 'f/c/e', 'f/e', 'f/a'] - const pExpected = ['p/e', 'p/d'] - const rootExpected = ['a/a', 'a/b', 'a/c/d', 'a/c/e', 'a/e', 'b/e', 'b/f', 'b/d', 'e', 'g'] - try { await runAll([ cb => writeFiles(drive, files, cb), cb => writeLinks(drive, links, cb), - cb => validateReaddir(t, drive, 'f', ['a', 'b', 'c/d', 'c/e', 'e'], { recursive: true }, cb), - cb => validateReaddir(t, drive, 'p', ['d', 'e'], { recursive: true }, cb), - cb => validateReaddir(t, drive, 'b', ['e', 'f', 'd'], { recursive: true }, cb), - cb => validateReaddir(t, drive, '', [...rootExpected, ...fExpected, ...pExpected], { recursive: true }, cb) + cb => validateReaddir(t, drive, 'f', ['a', 'b', 'c', 'e'], cb), + cb => validateReaddir(t, drive, 'b', ['e', 'f', 'd'], cb), + cb => validateReaddir(t, drive, '', ['a', 'b', 'e', 'f', 'p', 'g'], cb) ]) } catch (err) { t.fail(err) From 0721730fb3b326c0479fcbebc9c96b512407ad0b Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Thu, 18 Jul 2019 22:27:07 +0200 Subject: [PATCH 103/108] 10.0.0-rc9 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 178e7391..aee6d8a2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc8", + "version": "10.0.0-rc9", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From 7ae2da850071e8daecc2290c65a3ab8154339b5d Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 19 Jul 2019 00:40:14 +0200 Subject: [PATCH 104/108] Add secretKey to _getContent for mounts + add CORE_LOCK symbol --- index.js | 6 +++++- lib/content.js | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 02309957..f4c34987 100644 --- a/index.js +++ b/index.js @@ -185,13 +185,17 @@ class Hyperdrive extends EventEmitter { const existingContent = self._contentStates.get(db) if (existingContent) return process.nextTick(cb, null, existingContent) + // This should be a getter. + const mountMetadata = db._trie.feed + const mountContentKeyPair = mountMetadata.secretKey ? contentKeyPair(mountMetadata.secretKey) : {} + db.getMetadata((err, publicKey) => { if (err) return cb(err) return onkey(publicKey) }) function onkey (publicKey) { - const contentOpts = { key: publicKey, ...contentOptions(self, opts && opts.secretKey) } + const contentOpts = { key: publicKey, ...contentOptions(self, (opts && opts.secretKey) || mountContentKeyPair.secretKey) } const feed = self._corestore.get(contentOpts) feed.ready(err => { if (err) return cb(err) diff --git a/lib/content.js b/lib/content.js index 32cf5434..20f43678 100644 --- a/lib/content.js +++ b/lib/content.js @@ -1,6 +1,8 @@ const mutexify = require('mutexify') const sodium = require('sodium-universal') +const CONTENT_LOCK = Symbol('HyperdriveContentLock') + function contentKeyPair (secretKey) { let seed = Buffer.allocUnsafe(sodium.crypto_sign_SEEDBYTES) let context = Buffer.from('hyperdri', 'utf8') // 8 byte context @@ -29,10 +31,10 @@ function contentOptions (self, secretKey) { class ContentState { constructor (feed) { this.feed = (feed instanceof ContentState) ? feed.feed : feed - this._lock = mutexify() + if (!this.feed[CONTENT_LOCK]) this.feed[CONTENT_LOCK] = mutexify() } lock (cb) { - return this._lock(cb) + return this.feed[CONTENT_LOCK](cb) } } From 59a8e734fa04b504d68f2821aea9203cbbb4b0f4 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Fri, 19 Jul 2019 00:49:04 +0200 Subject: [PATCH 105/108] 10.0.0-rc10 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index aee6d8a2..7bd0ff71 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc9", + "version": "10.0.0-rc10", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": { From 1ef6993b0aa38c6328d985c10f9efa50bd9a3e3d Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 30 Jul 2019 14:38:54 +0200 Subject: [PATCH 106/108] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c41d8c00..7baa24d6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Hyperdrive is a secure, real time distributed file system ``` js -npm install hyperdrive +npm install hyperdrive@prerelease ``` [![Build Status](https://travis-ci.org/mafintosh/hyperdrive.svg?branch=master)](https://travis-ci.org/mafintosh/hyperdrive) From e2d88960e5c4ed2ded4dd7b2150fb69e4056a237 Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 30 Jul 2019 14:39:56 +0200 Subject: [PATCH 107/108] Add isLocked to content state --- index.js | 1 + lib/content.js | 3 +++ 2 files changed, 4 insertions(+) diff --git a/index.js b/index.js index f4c34987..17603995 100644 --- a/index.js +++ b/index.js @@ -380,6 +380,7 @@ class Hyperdrive extends EventEmitter { if (err && (err.errno !== 2)) return proxy.destroy(err) this._getContent(trie, (err, contentState) => { if (err) return proxy.destroy(err) + if (opts.wait === false && contentState.isLocked()) return cb(new Error('Content is locked.')) contentState.lock(_release => { release = _release append(contentState) diff --git a/lib/content.js b/lib/content.js index 20f43678..2ad0853a 100644 --- a/lib/content.js +++ b/lib/content.js @@ -36,6 +36,9 @@ class ContentState { lock (cb) { return this.feed[CONTENT_LOCK](cb) } + isLocked () { + return this.feed[CONTENT_LOCK].locked + } } module.exports = { From 933e45e85df8b9272ffb632f4d1b041c922bd01b Mon Sep 17 00:00:00 2001 From: Andrew Osheroff Date: Tue, 30 Jul 2019 14:42:10 +0200 Subject: [PATCH 108/108] 10.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7bd0ff71..9d3e7265 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hyperdrive", - "version": "10.0.0-rc10", + "version": "10.0.0", "description": "Hyperdrive is a secure, real time distributed file system", "main": "index.js", "scripts": {