Skip to content
This repository has been archived by the owner on Jul 3, 2019. It is now read-only.

Commit

Permalink
feat(opts): use figgy-pudding for opts (#128)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zkat
zkat authored Apr 9, 2018
1 parent 529f347 commit 33d4eed
Show file tree
Hide file tree
Showing 9 changed files with 862 additions and 800 deletions.
22 changes: 15 additions & 7 deletions get.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

const BB = require('bluebird')

const figgyPudding = require('figgy-pudding')
const fs = require('fs')
const index = require('./lib/entry-index')
const memo = require('./lib/memoization')
Expand All @@ -10,14 +11,20 @@ const pipeline = require('mississippi').pipeline
const read = require('./lib/content/read')
const through = require('mississippi').through

const GetOpts = figgyPudding({
integrity: {},
memoize: {},
size: {}
})

module.exports = function get (cache, key, opts) {
return getData(false, cache, key, opts)
}
module.exports.byDigest = function getByDigest (cache, digest, opts) {
return getData(true, cache, digest, opts)
}
function getData (byDigest, cache, key, opts) {
opts = opts || {}
opts = GetOpts(opts)
const memoized = (
byDigest
? memo.get.byDigest(cache, key, opts)
Expand Down Expand Up @@ -58,7 +65,7 @@ function getData (byDigest, cache, key, opts) {

module.exports.stream = getStream
function getStream (cache, key, opts) {
opts = opts || {}
opts = GetOpts(opts)
let stream = through()
const memoized = memo.get(cache, key, opts)
if (memoized && opts.memoize !== false) {
Expand Down Expand Up @@ -91,7 +98,6 @@ function getStream (cache, key, opts) {
} else {
memoStream = through()
}
opts.size = opts.size == null ? entry.size : opts.size
stream.emit('metadata', entry.metadata)
stream.emit('integrity', entry.integrity)
stream.emit('size', entry.size)
Expand All @@ -101,7 +107,9 @@ function getStream (cache, key, opts) {
ev === 'size' && cb(entry.size)
})
pipe(
read.readStream(cache, entry.integrity, opts),
read.readStream(cache, entry.integrity, opts.concat({
size: opts.size == null ? entry.size : opts.size
})),
memoStream,
stream
)
Expand All @@ -111,7 +119,7 @@ function getStream (cache, key, opts) {

module.exports.stream.byDigest = getStreamDigest
function getStreamDigest (cache, integrity, opts) {
opts = opts || {}
opts = GetOpts(opts)
const memoized = memo.get.byDigest(cache, integrity, opts)
if (memoized && opts.memoize !== false) {
const stream = through()
Expand Down Expand Up @@ -143,7 +151,7 @@ function getStreamDigest (cache, integrity, opts) {

module.exports.info = info
function info (cache, key, opts) {
opts = opts || {}
opts = GetOpts(opts)
const memoized = memo.get(cache, key, opts)
if (memoized && opts.memoize !== false) {
return BB.resolve(memoized.entry)
Expand All @@ -161,7 +169,7 @@ module.exports.copy.byDigest = function cpDigest (cache, digest, dest, opts) {
return copy(true, cache, digest, dest, opts)
}
function copy (byDigest, cache, key, dest, opts) {
opts = opts || {}
opts = GetOpts(opts)
if (read.copy) {
return (
byDigest ? BB.resolve(null) : index.find(cache, key, opts)
Expand Down
11 changes: 8 additions & 3 deletions lib/content/read.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
const BB = require('bluebird')

const contentPath = require('./path')
const figgyPudding = require('figgy-pudding')
const fs = require('graceful-fs')
const PassThrough = require('stream').PassThrough
const pipe = BB.promisify(require('mississippi').pipe)
Expand All @@ -11,9 +12,13 @@ const Y = require('../util/y.js')

BB.promisifyAll(fs)

const ReadOpts = figgyPudding({
size: {}
})

module.exports = read
function read (cache, integrity, opts) {
opts = opts || {}
opts = ReadOpts(opts)
return pickContentSri(cache, integrity).then(content => {
const sri = content.sri
const cpath = contentPath(cache, sri)
Expand All @@ -32,7 +37,7 @@ function read (cache, integrity, opts) {
module.exports.stream = readStream
module.exports.readStream = readStream
function readStream (cache, integrity, opts) {
opts = opts || {}
opts = ReadOpts(opts)
const stream = new PassThrough()
pickContentSri(
cache, integrity
Expand All @@ -56,7 +61,7 @@ if (fs.copyFile) {
module.exports.copy = copy
}
function copy (cache, integrity, dest, opts) {
opts = opts || {}
opts = ReadOpts(opts)
return pickContentSri(cache, integrity).then(content => {
const sri = content.sri
const cpath = contentPath(cache, sri)
Expand Down
4 changes: 3 additions & 1 deletion lib/content/write.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ function write (cache, data, opts) {
if (typeof opts.size === 'number' && data.length !== opts.size) {
return BB.reject(sizeError(opts.size, data.length))
}
const sri = ssri.fromData(data, opts)
const sri = ssri.fromData(data, {
algorithms: opts.algorithms
})
if (opts.integrity && !ssri.checkData(data, opts.integrity, opts)) {
return BB.reject(checksumError(opts.integrity, sri))
}
Expand Down
10 changes: 9 additions & 1 deletion lib/entry-index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const BB = require('bluebird')

const contentPath = require('./content/path')
const crypto = require('crypto')
const figgyPudding = require('figgy-pudding')
const fixOwner = require('./util/fix-owner')
const fs = require('graceful-fs')
const hashToSegments = require('./util/hash-to-segments')
Expand All @@ -29,9 +30,16 @@ module.exports.NotFoundError = class NotFoundError extends Error {
}
}

const IndexOpts = figgyPudding({
metadata: {},
size: {},
uid: {},
gid: {}
})

module.exports.insert = insert
function insert (cache, key, integrity, opts) {
opts = opts || {}
opts = IndexOpts(opts)
const bucket = bucketPath(cache, key)
const entry = {
key,
Expand Down
12 changes: 10 additions & 2 deletions lib/util/tmp.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,21 @@

const BB = require('bluebird')

const figgyPudding = require('figgy-pudding')
const fixOwner = require('./fix-owner')
const path = require('path')
const rimraf = BB.promisify(require('rimraf'))
const uniqueFilename = require('unique-filename')

const TmpOpts = figgyPudding({
tmpPrefix: {},
uid: {},
gid: {}
})

module.exports.mkdir = mktmpdir
function mktmpdir (cache, opts) {
opts = opts || {}
opts = TmpOpts(opts)
const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
return fixOwner.mkdirfix(tmpTarget, opts.uid, opts.gid).then(() => {
return tmpTarget
Expand All @@ -22,11 +29,12 @@ function withTmp (cache, opts, cb) {
cb = opts
opts = null
}
opts = opts || {}
opts = TmpOpts(opts)
return BB.using(mktmpdir(cache, opts).disposer(rimraf), cb)
}

module.exports.fix = fixtmpdir
function fixtmpdir (cache, opts) {
opts = TmpOpts(opts)
return fixOwner(path.join(cache, 'tmp'), opts.uid, opts.gid)
}
37 changes: 25 additions & 12 deletions lib/verify.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
const BB = require('bluebird')

const contentPath = require('./content/path')
const figgyPudding = require('figgy-pudding')
const finished = BB.promisify(require('mississippi').finished)
const fixOwner = require('./util/fix-owner')
const fs = require('graceful-fs')
Expand All @@ -14,10 +15,22 @@ const ssri = require('ssri')

BB.promisifyAll(fs)

const VerifyOpts = figgyPudding({
concurrency: {
default: 20
},
filter: {},
log: {
default: { silly () {} }
},
uid: {},
gid: {}
})

module.exports = verify
function verify (cache, opts) {
opts = opts || {}
opts.log && opts.log.silly('verify', 'verifying cache at', cache)
opts = VerifyOpts(opts)
opts.log.silly('verify', 'verifying cache at', cache)
return BB.reduce([
markStartTime,
fixPerms,
Expand All @@ -40,7 +53,7 @@ function verify (cache, opts) {
})
}, {}).tap(stats => {
stats.runTime.total = stats.endTime - stats.startTime
opts.log && opts.log.silly('verify', 'verification finished for', cache, 'in', `${stats.runTime.total}ms`)
opts.log.silly('verify', 'verification finished for', cache, 'in', `${stats.runTime.total}ms`)
})
}

Expand All @@ -53,7 +66,7 @@ function markEndTime (cache, opts) {
}

function fixPerms (cache, opts) {
opts.log && opts.log.silly('verify', 'fixing cache permissions')
opts.log.silly('verify', 'fixing cache permissions')
return fixOwner.mkdirfix(cache, opts.uid, opts.gid).then(() => {
// TODO - fix file permissions too
return fixOwner.chownr(cache, opts.uid, opts.gid)
Expand All @@ -70,11 +83,11 @@ function fixPerms (cache, opts) {
// 5. If content is not marked as live, rimraf it.
//
function garbageCollect (cache, opts) {
opts.log && opts.log.silly('verify', 'garbage collecting content')
opts.log.silly('verify', 'garbage collecting content')
const indexStream = index.lsStream(cache)
const liveContent = new Set()
indexStream.on('data', entry => {
if (opts && opts.filter && !opts.filter(entry)) { return }
if (opts.filter && !opts.filter(entry)) { return }
liveContent.add(entry.integrity.toString())
})
return finished(indexStream).then(() => {
Expand Down Expand Up @@ -117,7 +130,7 @@ function garbageCollect (cache, opts) {
})
})
}
}, {concurrency: opts.concurrency || 20}))
}, {concurrency: opts.concurrency}))
})
})
}
Expand All @@ -141,7 +154,7 @@ function verifyContent (filepath, sri) {
}

function rebuildIndex (cache, opts) {
opts.log && opts.log.silly('verify', 'rebuilding index')
opts.log.silly('verify', 'rebuilding index')
return index.ls(cache).then(entries => {
const stats = {
missingContent: 0,
Expand All @@ -153,7 +166,7 @@ function rebuildIndex (cache, opts) {
if (entries.hasOwnProperty(k)) {
const hashed = index._hashKey(k)
const entry = entries[k]
const excluded = opts && opts.filter && !opts.filter(entry)
const excluded = opts.filter && !opts.filter(entry)
excluded && stats.rejectedEntries++
if (buckets[hashed] && !excluded) {
buckets[hashed].push(entry)
Expand All @@ -170,7 +183,7 @@ function rebuildIndex (cache, opts) {
}
return BB.map(Object.keys(buckets), key => {
return rebuildBucket(cache, buckets[key], stats, opts)
}, {concurrency: opts.concurrency || 20}).then(() => stats)
}, {concurrency: opts.concurrency}).then(() => stats)
})
}

Expand All @@ -195,13 +208,13 @@ function rebuildBucket (cache, bucket, stats, opts) {
}

function cleanTmp (cache, opts) {
opts.log && opts.log.silly('verify', 'cleaning tmp directory')
opts.log.silly('verify', 'cleaning tmp directory')
return rimraf(path.join(cache, 'tmp'))
}

function writeVerifile (cache, opts) {
const verifile = path.join(cache, '_lastverified')
opts.log && opts.log.silly('verify', 'writing verifile to ' + verifile)
opts.log.silly('verify', 'writing verifile to ' + verifile)
return fs.writeFileAsync(verifile, '' + (+(new Date())))
}

Expand Down
Loading

0 comments on commit 33d4eed

Please sign in to comment.