From 06a35f907407682895deb27b6d1b5647c34bcf42 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Wed, 11 Dec 2019 17:08:02 +0000 Subject: [PATCH] feat: support unixfs metadata and formatting it (#14) Adds mode and mtime properties to normalised .add inputs, also adds functions for turning metadata into strings for CLI use. BREAKING CHANGE: In order to support metadata on intermediate directories, globSource in this module will now emit directories and files where previously it only emitted files. --- package.json | 2 +- src/files/format-mode.js | 66 +++++++++ src/files/format-mtime.js | 19 +++ src/files/glob-source.js | 60 +++++++- src/files/normalise-input.js | 6 +- test/files/format-mode.spec.js | 58 ++++++++ test/files/format-mtime.spec.js | 15 ++ test/files/glob-source.spec.js | 190 +++++++++++++++++++------ test/files/normalise-input.spec.js | 2 +- test/fixtures/dir/file-2.js | 0 test/fixtures/dir/nested-dir/other.txt | 0 11 files changed, 367 insertions(+), 51 deletions(-) create mode 100644 src/files/format-mode.js create mode 100644 src/files/format-mtime.js create mode 100644 test/files/format-mode.spec.js create mode 100644 test/files/format-mtime.spec.js mode change 100644 => 100755 test/fixtures/dir/file-2.js create mode 100644 test/fixtures/dir/nested-dir/other.txt diff --git a/package.json b/package.json index 775e172..f66b50c 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ }, "devDependencies": { "aegir": "^20.3.0", - "async-iterator-all": "^1.0.0", + "it-all": "^1.0.1", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "dirty-chai": "^2.0.1", diff --git a/src/files/format-mode.js b/src/files/format-mode.js new file mode 100644 index 0000000..f25ae5c --- /dev/null +++ b/src/files/format-mode.js @@ -0,0 +1,66 @@ +'use strict' + +const S_ISUID = parseInt('4000', 8) // set UID bit +const S_ISGID = parseInt('2000', 8) // set-group-ID bit (see below) +const S_ISVTX = parseInt('1000', 8) // sticky bit (see below) +// const S_IRWXU = parseInt('700', 8) // mask for file owner permissions +const S_IRUSR = parseInt('400', 8) // owner has read permission +const S_IWUSR = parseInt('200', 8) // owner has write permission +const S_IXUSR = parseInt('100', 8) // owner has execute permission +// const S_IRWXG = parseInt('70', 8) // mask for group permissions +const S_IRGRP = parseInt('40', 8) // group has read permission +const S_IWGRP = parseInt('20', 8) // group has write permission +const S_IXGRP = parseInt('10', 8) // group has execute permission +// const S_IRWXO = parseInt('7', 8) // mask for permissions for others (not in group) +const S_IROTH = parseInt('4', 8) // others have read permission +const S_IWOTH = parseInt('2', 8) // others have write permission +const S_IXOTH = parseInt('1', 8) // others have execute permission + +function checkPermission (mode, perm, type, output) { + if ((mode & perm) === perm) { + output.push(type) + } else { + output.push('-') + } +} + +function formatMode (mode, isDirectory) { + const output = [] + + if (isDirectory) { + output.push('d') + } else { + output.push('-') + } + + checkPermission(mode, S_IRUSR, 'r', output) + checkPermission(mode, S_IWUSR, 'w', output) + + if ((mode & S_ISUID) === S_ISUID) { + output.push('s') + } else { + checkPermission(mode, S_IXUSR, 'x', output) + } + + checkPermission(mode, S_IRGRP, 'r', output) + checkPermission(mode, S_IWGRP, 'w', output) + + if ((mode & S_ISGID) === S_ISGID) { + output.push('s') + } else { + checkPermission(mode, S_IXGRP, 'x', output) + } + + checkPermission(mode, S_IROTH, 'r', output) + checkPermission(mode, S_IWOTH, 'w', output) + + if ((mode & S_ISVTX) === S_ISVTX) { + output.push('t') + } else { + checkPermission(mode, S_IXOTH, 'x', output) + } + + return output.join('') +} + +module.exports = formatMode diff --git a/src/files/format-mtime.js b/src/files/format-mtime.js new file mode 100644 index 0000000..5ce121e --- /dev/null +++ b/src/files/format-mtime.js @@ -0,0 +1,19 @@ +'use strict' + +function formatMtime (mtime) { + if (mtime === undefined) { + return '-' + } + + return new Date(mtime * 1000).toLocaleDateString(Intl.DateTimeFormat().resolvedOptions().locale, { + year: 'numeric', + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + timeZoneName: 'short' + }) +} + +module.exports = formatMtime diff --git a/src/files/glob-source.js b/src/files/glob-source.js index e7e7b58..d6fbf0d 100644 --- a/src/files/glob-source.js +++ b/src/files/glob-source.js @@ -15,6 +15,10 @@ const kindOf = require('kind-of') * @param {Boolean} [options.hidden] Include .dot files in matched paths * @param {Array} [options.ignore] Glob paths to ignore * @param {Boolean} [options.followSymlinks] follow symlinks +* @param {Boolean} [options.preserveMode] preserve mode +* @param {Boolean} [options.preserveMtime] preserve mtime +* @param {Boolean} [options.mode] mode to use - if preserveMode is true this will be ignored +* @param {Boolean} [options.mtime] mtime to use - if preserveMtime is true this will be ignored * @yields {Object} File objects in the form `{ path: String, content: AsyncIterator }` */ module.exports = async function * globSource (paths, options) { @@ -47,21 +51,49 @@ module.exports = async function * globSource (paths, options) { const stat = await fs.stat(absolutePath) const prefix = Path.dirname(absolutePath) - for await (const entry of toGlobSource({ path, type: stat.isDirectory() ? 'dir' : 'file', prefix }, globSourceOptions)) { - yield entry + let mode = options.mode + + if (options.preserveMode) { + mode = stat.mode + } + + let mtime = options.mtime + + if (options.preserveMtime) { + mtime = parseInt(stat.mtimeMs / 1000) + } + + if (stat.isDirectory()) { + yield { + path: `/${Path.basename(path)}`, + mode, + mtime + } } + + yield * toGlobSource({ + path, + type: stat.isDirectory() ? 'dir' : 'file', + prefix, + mode, + mtime, + preserveMode: options.preserveMode, + preserveMtime: options.preserveMtime + }, globSourceOptions) } } -async function * toGlobSource ({ path, type, prefix }, options) { +async function * toGlobSource ({ path, type, prefix, mode, mtime, preserveMode, preserveMtime }, options) { options = options || {} const baseName = Path.basename(path) if (type === 'file') { yield { - path: baseName.replace(prefix, ''), - content: fs.createReadStream(Path.isAbsolute(path) ? path : Path.join(process.cwd(), path)) + path: `/${baseName.replace(prefix, '')}`, + content: fs.createReadStream(Path.isAbsolute(path) ? path : Path.join(process.cwd(), path)), + mode, + mtime } return @@ -77,15 +109,29 @@ async function * toGlobSource ({ path, type, prefix }, options) { const globOptions = Object.assign({}, options.glob, { cwd: path, - nodir: true, + nodir: false, realpath: false, absolute: true }) for await (const p of glob(path, '**/*', globOptions)) { + const stat = await fs.stat(p) + + if (preserveMode || preserveMtime) { + if (preserveMode) { + mode = stat.mode + } + + if (preserveMtime) { + mtime = parseInt(stat.mtimeMs / 1000) + } + } + yield { path: toPosix(p.replace(prefix, '')), - content: fs.createReadStream(p) + content: stat.isFile() ? fs.createReadStream(p) : undefined, + mode, + mtime } } } diff --git a/src/files/normalise-input.js b/src/files/normalise-input.js index 531402f..c821ac4 100644 --- a/src/files/normalise-input.js +++ b/src/files/normalise-input.js @@ -211,7 +211,11 @@ module.exports = function normaliseInput (input) { } function toFileObject (input) { - const obj = { path: input.path || '' } + const obj = { + path: input.path || '', + mode: input.mode, + mtime: input.mtime + } if (input.content) { obj.content = toAsyncIterable(input.content) diff --git a/test/files/format-mode.spec.js b/test/files/format-mode.spec.js new file mode 100644 index 0000000..7c1bb6c --- /dev/null +++ b/test/files/format-mode.spec.js @@ -0,0 +1,58 @@ +'use strict' + +/* eslint-env mocha */ +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const formatMode = require('../../src/files/format-mode') + +chai.use(dirtyChai) +const expect = chai.expect + +describe('format-mode', function () { + it('formats mode for directories', function () { + expect(formatMode(parseInt('0777', 8), true)).to.equal('drwxrwxrwx') + }) + + it('formats mode for files', function () { + expect(formatMode(parseInt('0777', 8), false)).to.equal('-rwxrwxrwx') + }) + + it('setgid, setuid and stick bit', function () { + expect(formatMode(parseInt('1777', 8), false)).to.equal('-rwxrwxrwt') + expect(formatMode(parseInt('2777', 8), false)).to.equal('-rwxrwsrwx') + expect(formatMode(parseInt('4777', 8), false)).to.equal('-rwsrwxrwx') + expect(formatMode(parseInt('5777', 8), false)).to.equal('-rwsrwxrwt') + expect(formatMode(parseInt('6777', 8), false)).to.equal('-rwsrwsrwx') + expect(formatMode(parseInt('7777', 8), false)).to.equal('-rwsrwsrwt') + }) + + it('formats user', function () { + expect(formatMode(parseInt('0100', 8), false)).to.equal('---x------') + expect(formatMode(parseInt('0200', 8), false)).to.equal('--w-------') + expect(formatMode(parseInt('0300', 8), false)).to.equal('--wx------') + expect(formatMode(parseInt('0400', 8), false)).to.equal('-r--------') + expect(formatMode(parseInt('0500', 8), false)).to.equal('-r-x------') + expect(formatMode(parseInt('0600', 8), false)).to.equal('-rw-------') + expect(formatMode(parseInt('0700', 8), false)).to.equal('-rwx------') + }) + + it('formats group', function () { + expect(formatMode(parseInt('0010', 8), false)).to.equal('------x---') + expect(formatMode(parseInt('0020', 8), false)).to.equal('-----w----') + expect(formatMode(parseInt('0030', 8), false)).to.equal('-----wx---') + expect(formatMode(parseInt('0040', 8), false)).to.equal('----r-----') + expect(formatMode(parseInt('0050', 8), false)).to.equal('----r-x---') + expect(formatMode(parseInt('0060', 8), false)).to.equal('----rw----') + expect(formatMode(parseInt('0070', 8), false)).to.equal('----rwx---') + }) + + it('formats other', function () { + expect(formatMode(parseInt('0001', 8), false)).to.equal('---------x') + expect(formatMode(parseInt('0002', 8), false)).to.equal('--------w-') + expect(formatMode(parseInt('0003', 8), false)).to.equal('--------wx') + expect(formatMode(parseInt('0004', 8), false)).to.equal('-------r--') + expect(formatMode(parseInt('0005', 8), false)).to.equal('-------r-x') + expect(formatMode(parseInt('0006', 8), false)).to.equal('-------rw-') + expect(formatMode(parseInt('0007', 8), false)).to.equal('-------rwx') + }) +}) diff --git a/test/files/format-mtime.spec.js b/test/files/format-mtime.spec.js new file mode 100644 index 0000000..d7a6eae --- /dev/null +++ b/test/files/format-mtime.spec.js @@ -0,0 +1,15 @@ +'use strict' + +/* eslint-env mocha */ +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const formatMtime = require('../../src/files/format-mtime') + +chai.use(dirtyChai) +const expect = chai.expect + +describe('format-mtime', function () { + it('formats mtime', function () { + expect((new Date(formatMtime(0))).getTime()).to.equal(0) + }) +}) diff --git a/test/files/glob-source.spec.js b/test/files/glob-source.spec.js index 214645c..38edbf2 100644 --- a/test/files/glob-source.spec.js +++ b/test/files/glob-source.spec.js @@ -5,41 +5,39 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const chaiAsPromised = require('chai-as-promised') const globSource = require('../../src/files/glob-source') -const all = require('async-iterator-all') +const all = require('it-all') const path = require('path') const { isNode } = require('../../src/env') +const fs = require('fs') chai.use(dirtyChai) chai.use(chaiAsPromised) const expect = chai.expect -describe('glob-source', () => { - it('single file, relative path', async function () { - if (!isNode) { - return this.skip() - } +function fixture (file) { + return path.resolve(path.join(__dirname, '..', 'fixtures', file)) +} - const result = await all(globSource(path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'file-0.html')))) +function findMode (file) { + return fs.statSync(fixture(file)).mode +} - expect(result.length).to.equal(1) - expect(result[0].path).to.equal('file-0.html') - }) +function findMtime (file) { + return parseInt(fs.statSync(fixture(file)).mtimeMs / 1000) +} - it('directory, relative path', async function () { +describe('glob-source', () => { + it('single file, relative path', async function () { if (!isNode) { return this.skip() } - const result = await all(globSource(path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), { - recursive: true - })) + const result = await all(globSource(fixture('file-0.html'))) - expect(result.length).to.equal(3) - expect(result[0].path).to.equal('/dir/file-1.txt') - expect(result[1].path).to.equal('/dir/file-2.js') - expect(result[2].path).to.equal('/dir/file-3.css') + expect(result.length).to.equal(1) + expect(result[0].path).to.equal('/file-0.html') }) it('single file, absolute path', async function () { @@ -47,10 +45,10 @@ describe('glob-source', () => { return this.skip() } - const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'file-0.html')))) + const result = await all(globSource(fixture('file-0.html'))) expect(result.length).to.equal(1) - expect(result[0].path).to.equal('file-0.html') + expect(result[0].path).to.equal('/file-0.html') }) it('directory, relative path', async function () { @@ -58,14 +56,18 @@ describe('glob-source', () => { return this.skip() } - const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), { + const result = await all(globSource(fixture('/dir'), { recursive: true })) - expect(result.length).to.equal(3) - expect(result[0].path).to.equal('/dir/file-1.txt') - expect(result[1].path).to.equal('/dir/file-2.js') - expect(result[2].path).to.equal('/dir/file-3.css') + expect(result).to.have.lengthOf(6) + expect(result).to.have.nested.property('[0].path', '/dir') + expect(result).to.not.have.nested.property('[0].content') + expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt') + expect(result).to.have.nested.property('[2].path', '/dir/file-2.js') + expect(result).to.have.nested.property('[3].path', '/dir/file-3.css') + expect(result).to.have.nested.property('[4].path', '/dir/nested-dir') + expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt') }) it('directory, hidden files', async function () { @@ -73,16 +75,19 @@ describe('glob-source', () => { return this.skip() } - const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), { + const result = await all(globSource(fixture('/dir'), { recursive: true, hidden: true })) - expect(result.length).to.equal(4) - expect(result[0].path).to.equal('/dir/.hidden.txt') - expect(result[1].path).to.equal('/dir/file-1.txt') - expect(result[2].path).to.equal('/dir/file-2.js') - expect(result[3].path).to.equal('/dir/file-3.css') + expect(result).to.have.lengthOf(7) + expect(result).to.have.nested.property('[0].path', '/dir') + expect(result).to.have.nested.property('[1].path', '/dir/.hidden.txt') + expect(result).to.have.nested.property('[2].path', '/dir/file-1.txt') + expect(result).to.have.nested.property('[3].path', '/dir/file-2.js') + expect(result).to.have.nested.property('[4].path', '/dir/file-3.css') + expect(result).to.have.nested.property('[5].path', '/dir/nested-dir') + expect(result).to.have.nested.property('[6].path', '/dir/nested-dir/other.txt') }) it('directory, ignore files', async function () { @@ -90,14 +95,17 @@ describe('glob-source', () => { return this.skip() } - const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), { + const result = await all(globSource(fixture('/dir'), { recursive: true, ignore: ['**/file-1.txt'] })) - expect(result.length).to.equal(2) - expect(result[0].path).to.equal('/dir/file-2.js') - expect(result[1].path).to.equal('/dir/file-3.css') + expect(result).to.have.lengthOf(5) + expect(result).to.have.nested.property('[0].path', '/dir') + expect(result).to.have.nested.property('[1].path', '/dir/file-2.js') + expect(result).to.have.nested.property('[2].path', '/dir/file-3.css') + expect(result).to.have.nested.property('[3].path', '/dir/nested-dir') + expect(result).to.have.nested.property('[4].path', '/dir/nested-dir/other.txt') }) it('multiple paths', async function () { @@ -106,13 +114,13 @@ describe('glob-source', () => { } const result = await all(globSource([ - path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-1.txt')), - path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-2.js')) + fixture('/dir/file-1.txt'), + fixture('/dir/file-2.js') ])) - expect(result.length).to.equal(2) - expect(result[0].path).to.equal('file-1.txt') - expect(result[1].path).to.equal('file-2.js') + expect(result).to.have.lengthOf(2) + expect(result).to.have.nested.property('[0].path', '/file-1.txt') + expect(result).to.have.nested.property('[1].path', '/file-2.js') }) it('requires recursive flag for directory', async function () { @@ -120,6 +128,106 @@ describe('glob-source', () => { return this.skip() } - await expect(all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir'))))).to.be.rejectedWith(/recursive option not set/) + await expect(all(globSource(fixture('/dir')))).to.be.rejectedWith(/recursive option not set/) + }) + + it('preserves mode for directories', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixture('/dir'), { + preserveMode: true, + recursive: true + })) + + expect(result).to.have.lengthOf(6) + expect(result).to.have.nested.property('[0].path', '/dir') + expect(result).to.have.nested.property('[0].mode', findMode('/dir')) + expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt') + expect(result).to.have.nested.property('[1].mode', findMode('/dir/file-1.txt')) + expect(result).to.have.nested.property('[2].path', '/dir/file-2.js') + expect(result).to.have.nested.property('[2].mode', findMode('/dir/file-2.js')) + expect(result).to.have.nested.property('[3].path', '/dir/file-3.css') + expect(result).to.have.nested.property('[3].mode', findMode('/dir/file-3.css')) + expect(result).to.have.nested.property('[4].path', '/dir/nested-dir') + expect(result).to.have.nested.property('[4].mode', findMode('/dir/nested-dir')) + expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt') + expect(result).to.have.nested.property('[5].mode', findMode('/dir/nested-dir/other.txt')) + }) + + it('overrides mode for directories', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixture('/dir'), { + recursive: true, + mode: 5 + })) + + expect(result).to.have.lengthOf(6) + expect(result).to.have.nested.property('[0].path', '/dir') + expect(result).to.have.nested.property('[0].mode', 5) + expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt') + expect(result).to.have.nested.property('[1].mode', 5) + expect(result).to.have.nested.property('[2].path', '/dir/file-2.js') + expect(result).to.have.nested.property('[2].mode', 5) + expect(result).to.have.nested.property('[3].path', '/dir/file-3.css') + expect(result).to.have.nested.property('[3].mode', 5) + expect(result).to.have.nested.property('[4].path', '/dir/nested-dir') + expect(result).to.have.nested.property('[4].mode', 5) + expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt') + expect(result).to.have.nested.property('[5].mode', 5) + }) + + it('preserves mtime for directories', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixture('/dir'), { + preserveMtime: true, + recursive: true + })) + + expect(result).to.have.lengthOf(6) + expect(result).to.have.nested.property('[0].path', '/dir') + expect(result).to.have.nested.property('[0].mtime', findMtime('/dir')) + expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt') + expect(result).to.have.nested.property('[1].mtime', findMtime('/dir/file-1.txt')) + expect(result).to.have.nested.property('[2].path', '/dir/file-2.js') + expect(result).to.have.nested.property('[2].mtime', findMtime('/dir/file-2.js')) + expect(result).to.have.nested.property('[3].path', '/dir/file-3.css') + expect(result).to.have.nested.property('[3].mtime', findMtime('/dir/file-3.css')) + expect(result).to.have.nested.property('[4].path', '/dir/nested-dir') + expect(result).to.have.nested.property('[4].mtime', findMtime('/dir/nested-dir')) + expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt') + expect(result).to.have.nested.property('[5].mtime', findMtime('/dir/nested-dir/other.txt')) + }) + + it('overrides mtime for directories', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(fixture('/dir'), { + recursive: true, + mtime: 5 + })) + + expect(result).to.have.lengthOf(6) + expect(result).to.have.nested.property('[0].path', '/dir') + expect(result).to.have.nested.property('[0].mtime', 5) + expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt') + expect(result).to.have.nested.property('[1].mtime', 5) + expect(result).to.have.nested.property('[2].path', '/dir/file-2.js') + expect(result).to.have.nested.property('[2].mtime', 5) + expect(result).to.have.nested.property('[3].path', '/dir/file-3.css') + expect(result).to.have.nested.property('[3].mtime', 5) + expect(result).to.have.nested.property('[4].path', '/dir/nested-dir') + expect(result).to.have.nested.property('[4].mtime', 5) + expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt') + expect(result).to.have.nested.property('[5].mtime', 5) }) }) diff --git a/test/files/normalise-input.spec.js b/test/files/normalise-input.spec.js index 2c11d75..f1e5f20 100644 --- a/test/files/normalise-input.spec.js +++ b/test/files/normalise-input.spec.js @@ -6,11 +6,11 @@ const dirtyChai = require('dirty-chai') const normalise = require('../../src/files/normalise-input') const { supportsFileReader } = require('../../src/supports') const { Buffer } = require('buffer') -const all = require('async-iterator-all') const pull = require('pull-stream') const Readable2 = require('readable-stream-2') const Readable3 = require('readable-stream') const ReadableNode = require('stream').Readable +const all = require('it-all') const globalThis = require('../../src/globalthis') chai.use(dirtyChai) diff --git a/test/fixtures/dir/file-2.js b/test/fixtures/dir/file-2.js old mode 100644 new mode 100755 diff --git a/test/fixtures/dir/nested-dir/other.txt b/test/fixtures/dir/nested-dir/other.txt new file mode 100644 index 0000000..e69de29