diff --git a/README.md b/README.md index ad80f8f..b9baa44 100644 --- a/README.md +++ b/README.md @@ -23,16 +23,58 @@ Alternatively, you can use the following environment variables: - `GATHERCONTENT_USER` - `GATHERCONTENT_AKEY` -### CLI +## Configuration -If you are using the command-line version of Metalsmith, you can install via npm, and then add the `metalsmith-jstransformer` key to your `metalsmith.json` file: - -```json -{ - "plugins": { - "metalsmith-gathercontent": {} - } -} +Right now this plugin is optional (though it passes the callback to the next metalsmith plugin if it finds no configuration) so some overriding needs to occur in `kalastatic.yaml`: +``` +# We maybe able to only pass the bits we need, but here's the full plugin list being overridden. +plugins: + - 'metalsmith-gathercontent' + # Load information from the environment variables. + - 'metalsmith-env' + # Define any global variables. + - 'metalsmith-define' + # Add .json metadata to each file. + - 'metalsmith-metadata-files' + # Add base, dir, ext, name, and href info to each file. + - 'metalsmith-paths' + # Load metadata info the metalsmith metadata object. + - 'metalsmith-metadata-convention' + # Concatenate any needed files. + - 'metalsmith-concat-convention' + # Load all collections. + - 'metalsmith-collections-convention' + # Bring in static assets. + - 'metalsmith-assets-convention' + # Ignore all partials and layouts. + - 'metalsmith-ignore' + # Load all Partials. + - 'metalsmith-jstransformer-partials' + # Render all content with JSTransformers. + - 'metalsmith-jstransformer' + # Clean URLs. + - 'metalsmith-clean-urls' + +# Allows changing some of the plugin options. +pluginOpts: + metalsmith-gathercontent: + verbose: true + saveJSON: true + authPath: _auth.json + filePath: src/assets/images/gathercontent + projectId: XXXXXX + mappings: + id: id + slug: _name + component: Meta_Component + title: Content_Title + tier: tier + type: _type + summary: Content_Summary + contents: Content_Content + hero__image: Content_Hero-Image + hero__image-alt: Content_Hero-Image-Alt + template: Meta_Component ``` ### JavaScript @@ -51,12 +93,13 @@ var gatherContent = require('metalsmith-gathercontent'); … .use(gatherContent({ authPath: '_auth.json', - projectId: 152172, + projectId: 000000, mappings: { id: 'id', slug: '_name', title: 'Content_Title', hero__image: 'Content_Hero-Image', + 'hero__image-alt': 'Content_Hero-Image-Alt', tier: 'tier', summary: 'Content_Summary', contents: 'Content_Content', @@ -64,7 +107,7 @@ var gatherContent = require('metalsmith-gathercontent'); article__image_gallery: 'Content_Image-Gallery' }, status: [ - 922006 + 000000 ] })) … @@ -76,23 +119,27 @@ The id of you Gather Content project. ### mappings Key value pairs to map variables from the hithercontent output. Where keys are the keys you want, and the values are what hithercontent is outputting. + This allows you to work with the Gather Content project as is. All additional keys are stored in a `fullData` object. -This plugin uses a "Meta" tab in gathercontent to store collections, and layouts. -Additionally if there are no mappings and a key `Content_Content` is present it will be automatically mapped to the `contents` property as a buffer. -As per hithercontent, keys within a Gather Content tab will be modified as follow `TabName_KeyName` +HitherContent outputs field keys using the following convention `${TabName}_${Dash-Delimited-Field-Name}` so in our case we currently follow the convention of: `Content` or `${Content-Descriptor}" (eg: `Content-Left`), `Meta` and `Social`. Thus a field called "CTA Image" in the "Content tab" on the GatherContent side would come in as `Content_CTA-Image`. + +Additionally if there are no mappings and a key `Content_Content` (a field named "Content" in the "Content" tab) is present it will be automatically mapped to the `contents` property as a markdown buffer and the key `fullData` contains the raw output from gatherContent key value pairs. ### status An array of Gather Content workflow status codes to filter against. -This way you can work with only "ready" content. +This way you can work with only "ready" content. When blank it ingests all content in a project regardless of status code. ### verbose More console.logs when set to true +### saveJSON +Outputs the raw Hithercontent json, as well the nested json we process through the mappings and 'children' parsing. This is useful for debugging. + ## Files and Images -Right now any key with `__image` in the index is processed as an image, and downloaded to `src/assets/gathercontent/` similarly for `__file` we will likely need to change this. Images that are arrays are stored as arrays. +Right now any post-mappings key ending in `__image` in the index is processed as an image, and downloaded to `src/assets/gathercontent/` (editable in `kalastatic.yaml`) additionally fully formed a11y images can be created with by naming fields with the following convention: the image is named `test__image` the alt text is `test__image-alt` so after processing `test__image` will hold `test__image.src`, `test__image.alt` and `test__image.origin`. We are working on a responsive image solution as we write this, as well as creating other conventions for certain common HTML primitives and microformats/schema. ## License MIT diff --git a/lib/index.js b/lib/index.js index dab346f..2ca43a8 100644 --- a/lib/index.js +++ b/lib/index.js @@ -9,8 +9,58 @@ const hithercontent = require('hithercontent') const _ = require('lodash') const request = require('request') const mkdirp = require('mkdirp') +const fileType = require('file-type') +const readChunk = require('read-chunk') -let filesToSave = [] +const defaultOpts = { + saveRemoteAssets: true, + useLocalData: false +} + +let filesToSave = {} + +/* To Title Case © 2018 David Gouch | https://github.com/gouch/to-title-case */ + +// eslint-disable-next-line no-extend-native +const toTitleCase = function (aString) { + 'use strict' + const smallWords = /^(a|an|and|as|at|but|by|en|for|if|in|nor|of|on|or|per|the|to|v.?|vs.?|via)$/i + const alphanumericPattern = /([A-Za-z0-9\u00C0-\u00FF])/ + const wordSeparators = /([ :–—-])/ + + return aString.split(wordSeparators) + .map((current, index, array) => { + if ( + /* Check for small words */ + current.search(smallWords) > -1 && + /* Skip first and last word */ + index !== 0 && + index !== array.length - 1 && + /* Ignore title end and subtitle start */ + array[index - 3] !== ':' && + array[index + 1] !== ':' && + /* Ignore small words that start a hyphenated phrase */ + (array[index + 1] !== '-' || + (array[index - 1] === '-' && array[index + 1] === '-')) + ) { + return current.toLowerCase() + } + + /* Ignore intentional capitalization */ + if (current.substr(1).search(/[A-Z]|\../) > -1) { + return current + } + /* Ignore URLs */ + if (array[index + 1] === ':' && array[index + 2] !== '') { + return current + } + /* Capitalize the first letter */ + return current.replace(alphanumericPattern, match => { + return match.toUpperCase() + }) + }) + .join('') +} const getItemById = (obj, id) => { for (let i = 0; i < obj.length; i++) { @@ -32,29 +82,34 @@ const buildFileName = (fileName, pId, items) => { return fileName } -const buildImage = (item, key, gcKey, opts) => { +const buildFile = (item, key, gcKey, opts) => { + opts = Object.assign(defaultOpts, opts) let val = item[gcKey] const filePath = opts.filePath.substr(opts.filePath.lastIndexOf('assets/'), opts.filePath.length) if (val.length > 1) { const files = [] val.forEach(file => { - const fileName = `/${filePath}/${file.substr(file.lastIndexOf('/') + 1, file.length)}` - const fileObj = { - origin: file, - src: fileName + if (file) { + const fileNameStub = file.substr(file.lastIndexOf('/') + 1, file.length) + const fileName = `/${filePath}/${fileNameStub}` + val = { + origin: file, + src: `${fileName}` + } + files.push(val) + filesToSave[fileNameStub] = val } - files.push(fileObj) - filesToSave.push(files) }) val = files } else if (val[0]) { const src = val[0] - const fileName = `/${filePath}/${src.substr(src.lastIndexOf('/') + 1, src.length)}` + const fileNameStub = src.substr(src.lastIndexOf('/') + 1, src.length) + const fileName = `/${filePath}/${fileNameStub}` val = { origin: src, - src: fileName + src: `${fileName}` } - filesToSave.push(val) + filesToSave[fileNameStub] = val } if (key.toLowerCase().indexOf('__image-alt') > -1 && item[gcKey + 'alt']) { val.alt = item[gcKey + '__alt'] @@ -69,14 +124,26 @@ const mapData = (obj, item, opts) => { if (val && gcKey === 'Content_Content') { val = Buffer.from(val) } else if (val && gcKey === '_name') { - val = _.kebabCase(val) - } else if (val && lowerKey.lastIndexOf('__image') > -1 && lowerKey.lastIndexOf('__image') === lowerKey.length - '__image'.length) { - val = buildImage(item, key, gcKey, opts) + val = toTitleCase(val) + } else if ( + val && + ( + (lowerKey.lastIndexOf('__image') > -1 && lowerKey.lastIndexOf('__image') === lowerKey.length - '__image'.length) || + (lowerKey.lastIndexOf('__file') > -1 && lowerKey.lastIndexOf('__file') === lowerKey.length - '__file'.length) + ) + ) { + val = buildFile(item, key, gcKey, opts) } - if (opts.verbose && opts.logMappings && val) { - console.log('\tmap to', obj._name, key, gcKey, val) + if (opts.verbose > 2 && opts.logMappings && val) { + console.log('Maping', obj._name, 'from', gcKey, 'to', key, 'with value', val) } if (val && val !== '') { + if (typeof val === 'string') { + val = val.trim() + } + if (gcKey === '_name') { + obj.slug = _.kebabCase(val) + } obj[key] = val } } @@ -93,7 +160,6 @@ const mapChildData = (item, opts) => { } const parseContent = (flatData, files, opts) => { - const virtualMdFiles = [] flatData.items.forEach(item => { const statusesToProcess = opts.status if (!opts.status || statusesToProcess.indexOf(Number(item._status.data.id)) > -1) { @@ -101,12 +167,14 @@ const parseContent = (flatData, files, opts) => { file.parentId = item._parent_id file.contents = Buffer.from('') const fileName = buildFileName(_.kebabCase(item._name), file.parentId, flatData.items).trim() + '.md' - file.fileName = fileName - files[fileName] = files[fileName] || {} + let page = {} + if (files[fileName]) { + page = files[fileName] + } if (Array.isArray(item.items) && item.items.length > 0) { - files[fileName].children = item.items + page.children = item.items if (opts.mappings) { - mapChildData(files[fileName], opts) + mapChildData(page, opts) } } if (item.Meta_Collection) { @@ -130,18 +198,18 @@ const parseContent = (flatData, files, opts) => { } if (item.Meta_Layout && item.Meta_Layout.trim() !== '') { file.layout = item.Meta_Layout.trim() - if (opts.verbose) { + if (opts.verbose > 2) { console.log('Applying layout', file.fileName, file.layout.trim()) } } - virtualMdFiles.push(file) - files[fileName] = Object.assign(files[fileName], file) - if (opts.verbose) { - console.log('metalsmith-gathercontent creating virtual markdown file:', fileName) + if ((opts.includeAsFile && opts.includeAsFile.includes(fileName)) || !opts.includeAsFile) { + files[fileName] = Object.assign(page, file) + if (opts.verbose > 0) { + console.log('metalsmith-gathercontent creating virtual markdown file:', fileName) + } } } }) - return virtualMdFiles } const flattenChildItems = (currentTier, flatData) => { @@ -162,25 +230,39 @@ const saveFiles = (opts, done) => { mkdirp.sync(`${process.cwd()}/${opts.filePath}`) request(origin).pipe(fs.createWriteStream(destFileName)).on('close', () => { filesSaved.push(src) - if (opts.verbose) { - console.log('metalsmith-gathercontent saved file:', destFileName) - } - if (filesSaved.length === filesToSave.length) { + const type = fileType(readChunk.sync(destFileName, 0, 4100)) + fs.rename(destFileName, `${destFileName}.${type.ext}`, err => { + if (err) { + console.log('Error:', err) + } + }) + console.log(`metalsmith-gathercontent saved file: ${destFileName} of type ${type.ext}, ${type.mime}`) + if (filesSaved.length === Object.entries(filesToSave).length) { done() } }) }) } +const postParsing = (filesToSave, flatData, metalsmith, done, opts) => { + metalsmith.metadata().gatherContent = { + flatData, + templateMap: opts.templateMappings + } + if (filesToSave && Object.entries(filesToSave).length > 0 && opts.saveRemoteAssets) { + saveFiles(opts, done) + } else { + done() + } +} + module.exports = function (opts) { return (files, metalsmith, done) => { - filesToSave = [] + filesToSave = {} if (!opts) { done() } else if (!opts.authPath) { - if (opts.verbose) { - console.log('Metalsmith GatherContent requires an _auth.json file to function properly passing callback to next plugin…') - } + console.log('Metalsmith GatherContent requires an _auth.json file to function properly passing callback to next plugin…') done() } // Retrieve the GatherContent authentication data. @@ -190,41 +272,44 @@ module.exports = function (opts) { } // Use auth.json when thhe environment variables arn't provided. if (!auth.user) { - auth = JSON.parse(fs.readFileSync(opts.authPath, {encoding: 'utf8'})) + auth = JSON.parse(fs.readFileSync(opts.authPath, { + encoding: 'utf8' + })) } - hithercontent.init(auth) - hithercontent.getProjectBranchWithFileInfo(opts.projectId, hithercontent.reduceItemToKVPairs, res => { - const flatData = {items: []} - const gatherContentData = res - if (opts.saveJSON) { - fs.writeFileSync('origin.json', JSON.stringify(res), 'utf8') - } - flattenChildItems(gatherContentData, flatData.items) + let gatherContentData + let flatData = { + items: [] + } + if (opts.useLocalData && fs.existsSync('parsed.json')) { + flatData = JSON.parse(fs.readFileSync('parsed.json', { + encoding: 'utf8' + })) parseContent(flatData, files, opts) - Object.entries(files).forEach(file => { - if (opts.verbose && opts.logFileContents) { - console.log('»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»') - console.log(file) - console.log('»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»') - console.log('') + postParsing(filesToSave, flatData, metalsmith, done, opts) + } else { + hithercontent.init(auth) + hithercontent.getProjectBranchWithFileInfo(opts.projectId, hithercontent.reduceItemToKVPairs, res => { + gatherContentData = res + if (opts.saveJSON) { + console.log('Saving hithercontent json file') + fs.writeFileSync('hithercontent.json', JSON.stringify(res), 'utf8') + } + flattenChildItems(gatherContentData, flatData.items) + parseContent(flatData, files, opts) + Object.entries(files).forEach(file => { + if (opts.verbose > 2 && Boolean(opts.logFileContents)) { + console.log('»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»') + console.log(file) + console.log('»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»»') + console.log('') + } + }) + if (opts.saveJSON) { + console.log('Saving parsed.json file') + fs.writeFileSync('parsed.json', JSON.stringify(flatData), 'utf8') } + postParsing(filesToSave, flatData, metalsmith, done, opts) }) - /* - This is just useful, it doesn't get parsed by metalsmith (thats virtualMdFiles) - but this data will continue to be available, for flexibility - */ - if (opts.saveJSON) { - fs.writeFileSync('flat.json', JSON.stringify(flatData), 'utf8') - } - metalsmith.metadata().gatherContent = { - flatData, - data: gatherContentData - } - if (filesToSave && filesToSave.length > 0) { - saveFiles(opts, done) - } else { - done() - } - }) + } } } diff --git a/package.json b/package.json index 923888f..90537f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "metalsmith-gathercontent", - "version": "1.4.1", + "version": "2.5.0", "description": "GatherContent metalsmith plugin using hithercontent", "main": "index.js", "directories": { @@ -26,10 +26,12 @@ "dependencies": { "base-64": "^0.1.0", "fetch": "^1.1.0", + "file-type": "^9.0.0", "fs": "0.0.1-security", "hithercontent": "^0.2.1", "lodash": "^4.17.10", "mkdirp": "^0.5.1", + "read-chunk": "^3.0.0", "node-fetch": "^2.2.0", "request": "^2.88.0" }, diff --git a/test/index.js b/test/index.js index 57dac5e..04785c1 100644 --- a/test/index.js +++ b/test/index.js @@ -106,7 +106,8 @@ setupTest('general', { setupTest('status-filtering', { pluginOpts: { 'metalsmith-gathercontent': { - verbose: 'true', + verbose: true, + logFileContents: true, authPath: '_auth.json', filePath: 'test/fixtures/status-filtering/src/assets/images/gathercontent', projectId: 152172,