From 05daec8fc904566631c2101484c893b2e3524f4a Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sat, 17 Nov 2018 20:35:16 -0800 Subject: [PATCH 1/3] Setting up spreadsheet CMS for 2019. --- .gitignore | 2 + .travis.yml | 14 +- plugins/nunjucks.js | 10 +- schedule.json | 1 - scripts/spreadsheet-import/credentials.js | 86 ++++++ .../image-utils/speaker-image.js | 84 ++++++ .../image-utils/sponsor-image.js | 61 +++++ scripts/spreadsheet-import/index.js | 249 ++++++++++++++++++ .../init-jwt-auth-client.js | 29 ++ .../spreadsheet-import/init-oauth2-client.js | 119 +++++++++ .../spreadsheet-import/process-schedule.js | 91 +++++++ scripts/spreadsheet-import/spreadsheet-api.js | 68 +++++ .../spreadsheet-import/spreadsheet-utils.js | 114 ++++++++ secrets.tar.enc | Bin 0 -> 2592 bytes templates/pages/about.html.njk | 52 +++- templates/partials/speaker-picture.html.njk | 11 + 16 files changed, 979 insertions(+), 12 deletions(-) delete mode 100644 schedule.json create mode 100644 scripts/spreadsheet-import/credentials.js create mode 100644 scripts/spreadsheet-import/image-utils/speaker-image.js create mode 100644 scripts/spreadsheet-import/image-utils/sponsor-image.js create mode 100644 scripts/spreadsheet-import/index.js create mode 100644 scripts/spreadsheet-import/init-jwt-auth-client.js create mode 100644 scripts/spreadsheet-import/init-oauth2-client.js create mode 100644 scripts/spreadsheet-import/process-schedule.js create mode 100644 scripts/spreadsheet-import/spreadsheet-api.js create mode 100644 scripts/spreadsheet-import/spreadsheet-utils.js create mode 100644 secrets.tar.enc create mode 100644 templates/partials/speaker-picture.html.njk diff --git a/.gitignore b/.gitignore index aaea2d9..81fa81a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ node_modules/ /.wwwtfrc /service-account-credentials.json /secrets.tar +/schedule.json +/contents/schedule-json.txt diff --git a/.travis.yml b/.travis.yml index 6124e2b..94da9af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,17 +7,17 @@ node_js: script: - npm run-script generate-locals - # - npm run-script generate-redirects + - npm run-script generate-redirects - npm run-script build # Because we build again, move the non-deploy build out of the way. - # - mv build test-build + - mv build test-build before_deploy: -# - openssl aes-256-cbc -K $encrypted_92a5b1b18fab_key -iv $encrypted_92a5b1b18fab_iv -in secrets.tar.enc -out secrets.tar -d -# - tar xvf secrets.tar -# - npm run-script ci:import -# - npm run-script build -# - node tests/post-build.js + - openssl aes-256-cbc -in secrets.tar.enc -out secrets.tar.out -d -pass "env:encrypted_client_secrets_password" + - tar xvf secrets.tar + - npm run-script ci:import + - npm run-script build + - node tests/post-build.js deploy: provider: pages diff --git a/plugins/nunjucks.js b/plugins/nunjucks.js index dc38b90..f3505e9 100644 --- a/plugins/nunjucks.js +++ b/plugins/nunjucks.js @@ -13,7 +13,7 @@ module.exports = function(env, callback) { extensions: {} }; Object.assign(env.config.locals, require('../locals-generated.json')); - env.config.locals.schedule = require('../schedule.json'); + // env.config.locals.schedule = require('../schedule.json'); env.config.locals.Date = Date; // Load the new nunjucks environment. @@ -47,14 +47,18 @@ module.exports = function(env, callback) { nenv.opts.autoescape = options.autoescape; class NunjucksTemplate extends env.TemplatePlugin { - constructor(template) { + constructor(template, filename) { super(); this.template = template; + this.filename = filename; } render(locals, callback) { try { let html = this.template.render(locals); + if (!html) { + throw new Error('Template render failed' + this.filename); + } html = minify(html, { removeAttributeQuotes: true, collapseWhitespace: true, @@ -69,7 +73,7 @@ module.exports = function(env, callback) { } static fromFile(filepath, callback) { - callback(null, new NunjucksTemplate(nenv.getTemplate(filepath.relative))); + callback(null, new NunjucksTemplate(nenv.getTemplate(filepath.relative), filepath.relative)); } } diff --git a/schedule.json b/schedule.json deleted file mode 100644 index 0967ef4..0000000 --- a/schedule.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/scripts/spreadsheet-import/credentials.js b/scripts/spreadsheet-import/credentials.js new file mode 100644 index 0000000..a777a6d --- /dev/null +++ b/scripts/spreadsheet-import/credentials.js @@ -0,0 +1,86 @@ +const path = require('path'); +const fs = require('fs'); +const credentialsCache = {}; + +function getCredentialsPath() { + return ( + process.env.WWWTF_CREDENTIALS_PATH || + path.resolve(process.env.HOME, '.wwwtf18') + ); +} + +/** + * Loads a private credentials-file. + * Files with sensitive information are stored in the directory ~/.wwwtf18 + * in JSON-Format. An alternative directory can be specified using the + * env-variable `WWWTF_CREDENTIALS_PATH`, but that is mostly intended + * to be used for CI/CD-servers and similar environments. + * @param {string} filename json-filename without any path. + * @param {boolean} ignoreMissing true to ignore missing files and return + * no content. + * @returns {*} the parsed content of the json-file + */ +function loadCredentials(filename, ignoreMissing = false) { + if (!credentialsCache[filename]) { + const credentialsFile = path.resolve(getCredentialsPath(), filename); + + if (ignoreMissing && !fs.existsSync(credentialsFile)) { + return null; + } + + try { + credentialsCache[filename] = require(credentialsFile); + } catch (err) { + console.error( + "šŸ” It appears that you don't have your credentials setup yet.\n" + + ' Please copy the file ' + + filename + + ' to\n ' + + getCredentialsPath() + + '\n to continue. Ask your coworkers if you never heard of that file.' + ); + + throw new Error(`failed to load ${credentialsFile}: ${err}`); + } + } + + return credentialsCache[filename]; +} + +/** + * Checks if credentials with the given filename exist. + * @param {string} filename file basename + * @return {boolean} - + */ +function hasCredentials(filename) { + const credentialsFile = path.resolve(getCredentialsPath(), filename); + + return fs.existsSync(credentialsFile); +} + +/** + * Stores credentials to a file in the credentials-store. + * @param {string} filename the file basename. + * @param {object} data the data to store. + */ +function storeCredentials(filename, data) { + credentialsCache[filename] = data; + + const credentialsFile = path.resolve(getCredentialsPath(), filename); + try { + fs.writeFileSync(credentialsFile, JSON.stringify(data)); + } catch (err) { + console.error( + `šŸ” Failed to write credentials to file ${credentialsFile}.` + ); + + throw new Error(`failed to write credentials: ${err}`); + } +} + +module.exports = { + getCredentialsPath, + loadCredentials, + hasCredentials, + storeCredentials +}; diff --git a/scripts/spreadsheet-import/image-utils/speaker-image.js b/scripts/spreadsheet-import/image-utils/speaker-image.js new file mode 100644 index 0000000..c8675c2 --- /dev/null +++ b/scripts/spreadsheet-import/image-utils/speaker-image.js @@ -0,0 +1,84 @@ +const fs = require('fs'); +const path = require('path'); +const fetch = require('node-fetch'); +const chalk = require('chalk'); +const imageType = require('image-type'); +const imageSize = require('image-size'); + +function getImageFilename(speaker, ext) { + let filename = speaker.firstname + '-' + speaker.lastname; + filename = filename.replace(/[^\w]/g, '-'); + filename = filename.replace(/--/g, '-').toLowerCase(); + + return filename + '.' + ext; +} + +function getLocalSpeakerImage(imagePath, speaker) { + if (!imagePath) { + return null; + } + + const filename = getImageFilename(speaker, 'jpg'); + const srcFilename = path.join(imagePath, filename); + const destFilename = path.join('contents/images/speaker', filename); + + if (fs.existsSync(srcFilename)) { + console.log(` --> image found in image-path:`, filename); + const buffer = fs.readFileSync(srcFilename); + const size = imageSize(buffer); + fs.writeFileSync(destFilename, buffer); + + return { + filename, + width: size.width, + height: size.height + }; + } + + return null; +} + +async function downloadSpeakerImage(speaker) { + const url = speaker.potraitImageUrl; + console.log('downloadImage', url); + if (!url) { + console.error(chalk.yellow('no image specified for ' + speaker.id)); + return {}; + } + + try { + const res = await fetch(url); + + if (!res.headers.get('content-type').startsWith('image')) { + console.error(chalk.red.bold(' !!! url is not an image', url)); + return {}; + } + + const buffer = await res.buffer(); + + const info = imageType(buffer); + if (!info) { + console.error(chalk.red.bold(' !!! no type-imformation for image', url)); + return {}; + } + + const size = imageSize(buffer); + const filename = getImageFilename(speaker, info.ext); + const fullPath = 'contents/images/speaker/' + filename; + + console.info(' --> image downloaded ', chalk.green(fullPath)); + fs.writeFileSync(fullPath, buffer); + + return { + filename, + width: size.width, + height: size.height + }; + } catch (err) { + console.error(chalk.red.bold(' !!! failed to download', url)); + console.error(err); + return {}; + } +} + +module.exports = {getLocalSpeakerImage, downloadSpeakerImage}; diff --git a/scripts/spreadsheet-import/image-utils/sponsor-image.js b/scripts/spreadsheet-import/image-utils/sponsor-image.js new file mode 100644 index 0000000..8926668 --- /dev/null +++ b/scripts/spreadsheet-import/image-utils/sponsor-image.js @@ -0,0 +1,61 @@ +const fs = require('fs'); +const path = require('path'); +const fetch = require('node-fetch'); +const chalk = require('chalk'); + +function getLocalSponsorImage(imagePath, sponsor) { + if (!imagePath) { + return null; + } + + const filename = sponsor.id + '.svg'; + const srcFilename = path.join(imagePath, filename); + const destFilename = path.join('contents/images/sponsor', filename); + + if (fs.existsSync(srcFilename)) { + console.log(` --> image found in image-path:`, filename); + const buffer = fs.readFileSync(srcFilename); + fs.writeFileSync(destFilename, buffer); + + return { + filename + }; + } + + return null; +} + +async function downloadSponsorImage(sponsor) { + const url = sponsor.logoUrl; + console.log('downloadImage', url); + if (!url) { + console.error(chalk.yellow('no image specified for ' + sponsor.id)); + return {}; + } + + try { + const res = await fetch(url); + + if (!res.headers.get('content-type').startsWith('image')) { + console.error(chalk.red.bold(' !!! url is not an image', url)); + return {}; + } + + const buffer = await res.buffer(); + const filename = sponsor.id + '.svg'; + const fullPath = `contents/images/sponsor/${filename}`; + + console.info(' --> image downloaded ', chalk.green(fullPath)); + fs.writeFileSync(fullPath, buffer); + + return { + filename + }; + } catch (err) { + console.error(chalk.red.bold(' !!! failed to download', url)); + console.error(err); + return {}; + } +} + +module.exports = {getLocalSponsorImage, downloadSponsorImage}; diff --git a/scripts/spreadsheet-import/index.js b/scripts/spreadsheet-import/index.js new file mode 100644 index 0000000..64e2dcf --- /dev/null +++ b/scripts/spreadsheet-import/index.js @@ -0,0 +1,249 @@ +const fs = require('fs'); +const path = require('path'); +const yaml = require('js-yaml'); +const wordwrap = require('wordwrap')(80); +const chalk = require('chalk'); +const program = require('commander'); +const mkdirp = require('mkdirp'); +const {promisify} = require('util'); +const {getSheetData} = require('./spreadsheet-api'); +const {processSheet, simplifySpreadsheetData} = require('./spreadsheet-utils'); +const {downloadSpeakerImage, getLocalSpeakerImage} = require('./image-utils/speaker-image'); +const {downloadSponsorImage, getLocalSponsorImage} = require('./image-utils/sponsor-image'); +const {processSchedule} = require('./process-schedule'); +const rimraf = promisify(require('rimraf')); +const timeout = promisify(setTimeout); + +// spreadsheet-format is illustrated here: +// https://docs.google.com/spreadsheets/d/14TQHTYePS0SAaXGRNF3zYXvvk8xz25CXW-uekQy4HAs/edit + +program + .description( + 'import speaker- and talk-data from the specified spreadheet and ' + + 'update the files in contents/speakers and contents/talks' + ) + .arguments('') + .action(spreadsheet => { + const rxSpreadsheetIdFromUrl = /^https:\/\/docs\.google\.com\/.*\/d\/([^/]+).*$/; + + program.spreadsheetId = spreadsheet; + + if (rxSpreadsheetIdFromUrl.test(spreadsheet)) { + program.spreadsheetId = spreadsheet.replace(rxSpreadsheetIdFromUrl, '$1'); + } + }) + .option( + '-p --production', + "run in production-mode (don't import unpublished items)" + ) + .option('-i --image-path ', 'alternative path to look for images') + .option('-C --no-cleanup', "don't run cleanup before import") + .parse(process.argv); + +const contentRoot = path.resolve(__dirname, '../../contents'); +const sheetParams = { + /*speakers: { + templateGlobals: { + template: 'pages/speaker.html.njk' + }, + dataFieldName: 'speaker', + contentPath: 'speakers' + }, + mcs: { + templateGlobals: { + template: 'pages/person.html.njk' + }, + dataFieldName: 'speaker', + contentPath: 'mcs' + }, + artists: { + templateGlobals: { + template: 'pages/person.html.njk' + }, + dataFieldName: 'speaker', + contentPath: 'artists' + },*/ + team: { + templateGlobals: { + template: 'pages/person.html.njk' + }, + dataFieldName: 'speaker', + contentPath: 'team' + }, + /*sponsors: { + templateGlobals: { + template: 'pages/sponsor.html.njk' + }, + dataFieldName: 'sponsor', + contentPath: 'sponsors' + }, + schedule: { + parseSchedule: true, + },*/ +}; + +const wwwtfrcFile = __dirname + '/../../.wwwtfrc'; +const hasRcFile = fs.existsSync(wwwtfrcFile); + +let rcFileParams = {}; +if (hasRcFile) { + rcFileParams = JSON.parse(fs.readFileSync(wwwtfrcFile)); +} + +const params = { + ...rcFileParams, + imagePath: program.imagePath, + doCleanup: program.cleanup, + publishedOnly: program.production || process.env.NODE_ENV === 'production' +}; +if (program.spreadsheetId) { + params.spreadsheetId = program.spreadsheetId; +} + +if (!params.spreadsheetId) { + console.log( + chalk.red.bold('A spreadsheet-id (or spreadsheet-url) is required.') + ); + program.outputHelp(); + process.exit(1); +} + +if (!hasRcFile) { + console.log('saving settings to', chalk.green('.wwwtfrc')); + fs.writeFileSync( + wwwtfrcFile, + JSON.stringify({spreadsheetId: params.spreadsheetId}, null, 2) + ); +} + +main(params).catch(err => console.error(err)); + +async function main(params) { + // ---- ensure the directories exist... + const requiredDirectories = ['team', 'speakers', 'talks', 'sponsors', 'images/speaker', 'images/sponsor']; + const requiredDirectoryPaths = requiredDirectories.map( + dir => `${__dirname}/../../contents/${dir}` + ); + const missingDirectories = requiredDirectoryPaths.filter( + dir => !fs.existsSync(dir) + ); + + if (!!missingDirectories.length) { + console.log(chalk.gray('creating missing directories...')); + missingDirectories.forEach(dir => mkdirp(dir)); + } + + // ---- cleanup... + if (params.doCleanup) { + console.log(chalk.gray('cleaning up...')); + + await Promise.all([ + rimraf(path.join(contentRoot, 'images/{speaker,sponsor}/*')), + rimraf(path.join(contentRoot, '{speakers,sponsors,talks}/*md')), + ]); + } + + // ---- fetch spreadsheet-data... + console.log(chalk.gray('loading spreadsheet data...')); + const sheets = simplifySpreadsheetData( + await getSheetData(params.spreadsheetId, { + readonly: true, + + async beforeOpenCallback(url) { + console.log( + chalk.white( + '\n\nšŸ” You first need to grant access to your ' + + 'google-spreadsheets to this program.\n An ' + + 'authorization-dialog will be ' + + 'opened in your browser in 5 seconds.\n\n' + ), + chalk.blue.underline(url) + ); + + return await timeout(5000); + } + }) + ); + + // ---- parse and generate markdown-files + console.log(chalk.gray('awesome, that worked.')); + Object.keys(sheets).forEach(sheetId => { + if (!sheetId) { + // Published pages create unnamed sheets. + return; + } + if (!sheetParams[sheetId]) { + console.log(chalk.red('Missing metadata for'), sheetId); + return; + } + const {templateGlobals, dataFieldName, contentPath, parseSchedule} = sheetParams[sheetId]; + if (parseSchedule) { + processSchedule(sheets[sheetId]); + return; + } + const records = processSheet(sheets[sheetId]); + + console.log(chalk.white('processing sheet %s'), chalk.yellow(sheetId)); + records + // filter unpublished records when not in dev-mode. + .filter(r => r.published || !params.publishedOnly) + + // render md-files + .forEach(async function(record) { + const filename = path.join(contentRoot, contentPath, `${record.id}.md`); + + let {content, ...data} = record; + let title = ''; + + if (!content) { + content = ' '; + } + + if (!data.name) { + data.name = data.firstname + ' ' + data.lastname; + } + + if (sheetId === 'sponsors') { + data.image = getLocalSponsorImage(params.imagePath, data); + title = data.name; + if (!data.image) { + try { + data.image = await downloadSponsorImage(data); + } catch (err) { + console.error('this is bad: ', err); + } + } + delete data.logoUrl; + } else { + data.image = getLocalSpeakerImage(params.imagePath, data); + title = `${data.name}: ${data.talkTitle}`; + if (!data.image) { + data.image = await downloadSpeakerImage(data); + } + + delete data.potraitImageUrl; + } + + const frontmatter = yaml.safeDump({ + ...templateGlobals, + title, + [dataFieldName]: data + }); + + console.log( + ' --> write markdown %s', + chalk.green(path.relative(process.cwd(), filename)) + ); + + const markdownContent = + '----\n\n' + + '# THIS FILE WAS GENERATED AUTOMATICALLY.\n' + + '# CHANGES MADE HERE WILL BE OVERWRITTEN.\n\n' + + frontmatter.trim() + + '\n\n----\n\n' + + wordwrap(content); + + fs.writeFileSync(filename, markdownContent); + }); + }); +} diff --git a/scripts/spreadsheet-import/init-jwt-auth-client.js b/scripts/spreadsheet-import/init-jwt-auth-client.js new file mode 100644 index 0000000..9cad43d --- /dev/null +++ b/scripts/spreadsheet-import/init-jwt-auth-client.js @@ -0,0 +1,29 @@ +const {promisify} = require('util'); +const google = require('googleapis'); +const {loadCredentials} = require('./credentials'); + +/** + * Initializes the auth-client. + * @param {string} scope the oauth-scope. + * @return {Promise} a promise that resolves when the authClient is ready + */ +async function initJWTAuthClient(scope) { + /* eslint-disable camelcase */ + const {client_email, private_key} = loadCredentials( + 'service-account-credentials.json' + ); + const client = new google.auth.JWT( + client_email, + null, + private_key, + scope, + null + ); + /* eslint-enable */ + + await promisify(client.authorize.bind(client))(); + + return client; +} + +module.exports = {initJWTAuthClient}; diff --git a/scripts/spreadsheet-import/init-oauth2-client.js b/scripts/spreadsheet-import/init-oauth2-client.js new file mode 100644 index 0000000..723e2e9 --- /dev/null +++ b/scripts/spreadsheet-import/init-oauth2-client.js @@ -0,0 +1,119 @@ +const crypto = require('crypto'); +const http = require('http'); +const {promisify} = require('util'); +const {parse: parseUrl} = require('url'); + +const open = require('open'); +const getPort = require('get-port'); +const google = require('googleapis'); + +const { + loadCredentials, + hasCredentials, + storeCredentials +} = require('./credentials'); + +/** + * Initialize the oauth-client for the specified scope. + * @param {string} scope the oauth-scope to request authorization for. + * @param {object} options additional options + * @return {Promise} the auth-client. + */ +async function initOAuth2Client(scope, options = {}) { + const clientSecret = loadCredentials('client_secret.json').installed; + + const port = await getPort(); + const auth = new google.auth.OAuth2( + clientSecret.client_id, + clientSecret.client_secret, + `http://localhost:${port}` + ); + + const md5 = crypto.createHash('md5'); + const scopeHash = md5.update(scope).digest('hex'); + const credentialsFile = `credentials-${scopeHash}.json`; + + if (hasCredentials(credentialsFile)) { + auth.credentials = loadCredentials(credentialsFile); + } else { + auth.credentials = await getCredentials(auth, scope, options); + storeCredentials(credentialsFile, auth.credentials); + } + + return auth; +} + +/** + * Retrieves the auth-client credentials. + * @param {google.auth.OAuth2} auth the OAuth2-instance. + * @param {string} scope the scope to get authorization for. + * @param {object} options additional options + * @param {function?} options.beforeOpenCallback an async function to be called + * with the authorization-url before the url is opened. + * @return {Promise} the credentials, including access_token, + * refresh_token and expiry_date. + */ +async function getCredentials(auth, scope, options = {}) { + const getToken = promisify(auth.getToken.bind(auth)); + + const url = auth.generateAuthUrl({ + access_type: 'offline', // eslint-disable-line camelcase + scope + }); + + const redirectUri = parseUrl(url, true).query.redirect_uri; + const port = parseUrl(redirectUri).port; + + if (options.beforeOpenCallback) { + await options.beforeOpenCallback(url); + } + + open(url); + + return await getToken(await receiveAuthorizationCode(port)); +} + +/** + * Starts an http-server to listen for the oauth2 redirectUri to be called + * containing the authorization-code. + * @param {number} port port-number for the http-server + * @return {Promise} the authorization-code. + */ +async function receiveAuthorizationCode(port) { + const server = http.createServer(); + const listen = promisify(server.listen.bind(server)); + + await listen(port, '127.0.0.1'); + + return new Promise((resolve, reject) => { + server.once('request', (request, response) => { + const {code} = parseUrl(request.url, true).query; + + if (!code) { + response.end( + '\n' + + '' + + '

Well, that\'s embarrassing.

' + + '

It won\'t be possible to spreadsheets without this ' + + ' authorization. Maybe try again?

' + + '' + ); + + reject(new Error('authorization failed.')); + } else { + response.end( + '\n' + + '' + + '

Perfect!

' + + '

You can now close this browser window.

' + + '' + ); + } + server.close(); + + resolve(parseUrl(request.url, true).query.code); + }); + }); +} + +module.exports = {initOAuth2Client}; diff --git a/scripts/spreadsheet-import/process-schedule.js b/scripts/spreadsheet-import/process-schedule.js new file mode 100644 index 0000000..b8c1fec --- /dev/null +++ b/scripts/spreadsheet-import/process-schedule.js @@ -0,0 +1,91 @@ +const fs = require('fs'); + +module.exports.processSchedule = function(sheet) { + const data = structureData(sheet); + const schedule = { + info: info(), + schedule: data, + } + const json = JSON.stringify(schedule, null, ' '); + console.info(json); + fs.writeFileSync('./schedule.json', json); + // Write with .txt filename, because wintersmith doesn't support serving + // files with the "magic" .json extension. + fs.writeFileSync('./contents/schedule-json.txt', json); +} + +const columns = [ + 'backtrack:startTime', 'backtrack:duration', 'backtrack:number', + '-', 'backtrack:who', 'backtrack:what', '-', + 'sidetrack:startTime', 'sidetrack:duration', 'sidetrack:number', + '-', 'sidetrack:who', 'sidetrack:what', '-', + 'community:startTime', 'community:what', 'community:detail', '-', + 'sponsor:startTime', 'sponsor:what', 'sponsor:detail' +]; + +const tracksMap = { + backtrack: 'Back Track', + sidetrack: 'Side Track', + community: 'Community Lounge', + sponsor: 'Sponsor Booth' +} + +function structureData(lessCrappyData) { + let day = 1; + const mergedRecords = {}; + + for (let row = 2, nRows = lessCrappyData.length; row < nRows; row++) { + + + if (!lessCrappyData[row]) { continue; } + + if (/Day 2:/.test(lessCrappyData[row][0])) { + day = 2; + } + + const tracks = {}; + for (let col = 0, nCols = lessCrappyData[row].length; col < nCols; col++) { + if (!columns[col] || columns[col] === '-') { continue; } + const [track, field] = columns[col].split(':'); + + + if (!tracks[track]) { + tracks[track] = { + day: day, + date: day == 1 ? '2018-06-02' : '2018-06-03', + track: tracksMap[track], + trackId: track + }; + } + tracks[track][field] = lessCrappyData[row][col]; + } + + Object.keys(tracks).forEach(track => { + if (!tracks[track].startTime || !tracks[track].what) { + return; + } + tracks[track].startTime = String(tracks[track].startTime).replace(':', '.'); + tracks[track].dateTime = tracks[track].date + ' ' + + tracks[track].startTime.replace('.', ':') + + ' GMT+0200'; + if (!mergedRecords[day]) { + mergedRecords[day] = {}; + } + if (!mergedRecords[day][tracks[track].startTime]) { + mergedRecords[day][tracks[track].startTime] = []; + } + mergedRecords[day][tracks[track].startTime].push(tracks[track]); + }); + } + + return mergedRecords; +} + +function info() { + const now = new Date(); + const conferenceDay = now < Date.parse('Sun Jun 02 2018 00:00:00 GMT+0200 (CEST)') ? 1 : 2; + return { + currentDay: conferenceDay, + generationTime: now.toString(), + }; +} \ No newline at end of file diff --git a/scripts/spreadsheet-import/spreadsheet-api.js b/scripts/spreadsheet-import/spreadsheet-api.js new file mode 100644 index 0000000..ad40de8 --- /dev/null +++ b/scripts/spreadsheet-import/spreadsheet-api.js @@ -0,0 +1,68 @@ +const google = require('googleapis'); + +const {initOAuth2Client} = require('./init-oauth2-client'); +const {initJWTAuthClient} = require('./init-jwt-auth-client'); +const {hasCredentials, getCredentialsPath} = require('./credentials'); + +const sheets = google.sheets('v4'); + +/** + * Loads the specified sheet via the spreadsheets-API and returns it's + * raw data. + * @param {string} documentId The id of the spreadsheets-document + * @param {string} sheetId The id of the worksheet within the document + * @returns {Promise.} The raw spreadsheet-data + */ +async function getSheetData(documentId, options = {}) { + const requestOptions = { + auth: await getAuthClient(options), + spreadsheetId: documentId, + includeGridData: true + }; + + return new Promise((resolve, reject) => { + sheets.spreadsheets.get(requestOptions, (err, result) => { + if (err) { + return reject(err); + } + return resolve(result); + }); + }); +} + +let clientPromise = null; + +/** + * Initializes the auth-client. The preferred method is to use personal oauth2, + * since it allows better management of permissions. Alternatively JWT + * (aka service accounts) is supported for automated build-environments. + * @return {Promise} a promise that resolves when the authClient is ready + */ +async function getAuthClient(options = {}) { + if (clientPromise) { + return clientPromise; + } + + const scope = + 'https://www.googleapis.com/auth/spreadsheets' + + (options.readonly ? '.readonly' : ''); + + if (hasCredentials('client_secret.json')) { + clientPromise = initOAuth2Client(scope, options); + } else if (hasCredentials('service-account-credentials.json')) { + clientPromise = initJWTAuthClient(scope); + } else { + console.error( + "šŸ” couldn't create an auth-client. Please make sure that your " + + ` credentials in ${getCredentialsPath()} are properly set up.` + ); + + throw new Error('failed to authorize'); + } + + return clientPromise; +} + +module.exports = { + getSheetData +}; diff --git a/scripts/spreadsheet-import/spreadsheet-utils.js b/scripts/spreadsheet-import/spreadsheet-utils.js new file mode 100644 index 0000000..7dc9f41 --- /dev/null +++ b/scripts/spreadsheet-import/spreadsheet-utils.js @@ -0,0 +1,114 @@ +const objectPath = require('object-path'); + +/** + * Simplifies data from the spreadsheets-API by reducing it to actual values. + * @param {Object} response - + * @return {Object} - + */ +function simplifySpreadsheetData(response) { + return response.sheets.reduce((sheets, {properties, data}) => { + sheets[properties.title] = data[0].rowData + .filter(row => row.values) + .map(row => + row.values.map(value => { + if (!value.effectiveValue) { + return null; + } else if (typeof value.effectiveValue.numberValue !== 'undefined') { + return value.effectiveValue.numberValue; + } else if (typeof value.effectiveValue.stringValue !== 'undefined') { + return value.effectiveValue.stringValue; + } + + throw new Error('neither numberValue nor stringValue exists'); + }) + ) + .filter(row => row.some(value => value !== null)); + + return sheets; + }, {}); +} + +/** + * Process (parse) data from a single sheet in the spreadsheet document. + * @param sheetData + */ +function processSheet(sheetData) { + const columnNames = sheetData[0]; + const columnTypes = sheetData[1]; + const bodyRows = sheetData.slice(2); + + const columns = parseColumnHeaders(columnNames, columnTypes); + + return bodyRows.map(parseDataRow.bind(null, columns)); +} + +/** + * Parses and validates the column-headers. + * @param columnNames + * @param columnTypes + * @return {Column[]} + */ +function parseColumnHeaders(columnNames, columnTypes) { + const columns = []; + const columnsByName = {}; + + for (let i = 0; i < columnNames.length; i++) { + const name = columnNames[i]; + const type = columnTypes[i]; + + if (!name || name.startsWith('//')) { + continue; + } + + const column = { + dataIndex: i, + name, + type + }; + + // validate: make sure there isn't already a column with the same fieldname + const conflictingColumn = columnsByName[name]; + if (conflictingColumn) { + throw new Error( + `āš ļø name-conflict: column "${column.name}" (cell ` + + `${String.fromCharCode(65 + i)}1) has the same ` + + `fieldname as column "${conflictingColumn.header}" + (cell ${String.fromCharCode(65 + conflictingColumn.dataIndex)}1)` + ); + } + + columnsByName[name] = column; + columns.push(column); + } + + return columns; +} + +/** + * Parses a single record from the spreadsheet. + * @param {Column[]} columns the column-specifications from the header + * @param {string[]} row the data-row from the spreadsheet + * @returns {Object} the parsed record + */ +function parseDataRow(columns, row) { + const record = {}; + + for (const column of columns) { + const {dataIndex, name, type} = column; + let value = row[dataIndex]; + + if (typeof value === 'undefined') { + continue; + } + + if (type === 'boolean') { + value = Boolean(value); + } + + objectPath.set(record, name, value); + } + + return record; +} + +module.exports = {simplifySpreadsheetData, processSheet}; diff --git a/secrets.tar.enc b/secrets.tar.enc new file mode 100644 index 0000000000000000000000000000000000000000..90a2b8563464cdd58248221112ea5164bac41413 GIT binary patch literal 2592 zcmV+*3g7ipVQh3|WM5yu2-i{#Sxaj$g5fiMjqph=VqnmdR4i}#U_k=r^@SYpoht`S z$+1dZvChR~3%HWfZyREq8Jkxg$kw1D)SxGgvfVPkENvx>BiNWzM0kN(Ee4k{y<7oe zK+ZB)1W3v_@^pjpU8k9Mu*pH95UZmcu;0`@vR7!d|agM31P3zxs8pe=kY9sf)&P`N~woA%Ob2$ zC*}=LJ+XTM(rBrm*Y@XGYH=^b{z#qmiy<%YEG>}jd=l@@foO1X>lVSUwQ(g7G~zx@ zR7%~}b?UO(+b?AU(;9zsn%SLQF{DvU&D5mU9D5q}tz{9r5GA*z;|f<#$c_BbU!Abp zf$(GsU|*r`rCzKU9*CM!=8m9J)N~-07uS#^7-UKMx*;7M2+shdw)Dcd{9Apw2l1Ep zEORfmh&thDmScb_NJgz2mKcgN;mv384?qW6b)dvi(+d^8YaP%erwMa@1!y}NXeUsk za0{TMWn+~t5ie)1t>+cRv$@{1aS-ozn6dOkRa)AOpxefmO_RV5j|Qv8T0?wA;&gA2 zz?=bWeJ&{`6L7N-yv2%7nLQEEA=Po=DfkqBl7&#BabJ?DFI5m+3yaH*D2UJtS5cH`&p}N7&2%u4>8Vb=!Xzp(G3jrDX+M}D z=d>E#^(bGS`_Od|AT6Rw2J6lmFfZ%+m~#iInUztG%kftbyF>sezM)(wt**I&=DINL zxew53ZVZ_s&dMKJNJszi>^oI{vp~6l;Mu{rKPa7BN1ZqpM2DZ4a3i~oNuK0loLlDU zZ_-=9V%*zD5eMwMxbhSe!v~+U>6pVDl3LvCL4=6~6w4S+{Y!)gadXS}&mPIwWRR2s zm{eqo0KU{RrM42m6>h-GI3v2z=DkuqPloggkCZg_$L|Y6d$V@HwI}zNU37_4Tv!Rp zD3p~X%3b5aWsM@$=($Vkl*O)f9N#5_yjX55&N8iy8-nf-zcqqZ0aQ~$aK+l8y zdM6kmwSD1OTVD@<0~vkmj)rfAs^1eI1Z7YKNCC1EBo*-AIO*6pFT&rPRGOV(2v2rS zh@&78J!C|mn6HdOU%KQVCQc*sf(>Y{Q3QUjaBv>ml9A{GJM{uDxgXlM7Q6A$}_7W*$jp#5aVH+LEWN9rO}w#&^qQnZFieJfqP!(gI0=R zDnu4!Hj5-P=JnjaBxsv$Z_v#v=j z-;Ag-J%I7T3a-n&qNH){sZ*@%l7yM^%HgRURq(gdf4nOp~ck z+;huO+9QT#6Cx+{S|0bm%!ZfJvC9qDhlReQsSFTm$u~W&;KUuU<}c9Eondw;PwK)~ zH)?(@QHFq%$IQ~$-CkgR&&zK`~0UI(mv7x|K$#S$jazCeMQUKk#Rjpu{LI4V%e!ZD#DAsCjkxs_&7fVkOdxg|d zn5X(s(LPCAdb6E#UC2D~%!)gQVJYqevzytbz+Uv?x_CidqCtlzu+=Mxz|tvAsHm56 zOQ;OdAap^G)B2>((Lm6OsvaI+`PJ|LhXDQ({pAeI4m!q`xm1KjtGN#Lq^)Op{4^;7 zm1ON!Ldu%YWq!?#@X%VtPq6kqr2sU%{|9Yj`}{2uvif@{BJOqG9p$TytOj$rhmgEz zkOtYJlbqoUP6+T8IXf@J9qeF1YmOLu>Ylbj@?4*!clgN#I?q+1j)61F6(IAg(qYMK zLBQ%Iw>PT|=3>Y-XpmR*&U-Z_#svg=i*18DkTuL&J#_dGwRQ&UuFoRaK^~cKYiJn> zc(F$vNt{%Hy*2pFl|soYaYLO?F^2q;l-F~Pc4;;jd*qd%HvxuXA~LMMfSbAm|EE!N+NOJvGIMBS`epjO!+W^H>U!y(EU0+L)5Rk=JaYgM8+2BuMRC@HW|bm15b21)3|P8o zLy(5cj`E~*V$2WK$0&jk#rVZT1@@M|Y6l>DLW8RSN zE;KU~I6)a?DS_^)QGqBAuoV|nlDu$+g(O+v>*c4ff-J6fQdNkfm`MiJPo~uC;Kpcp zMk!CACXix@y&9&~j)SCVhX3ueQKA+(33ieeFqiubeq}tU^fY|R79|Qe>k0uQy2h;x z&4O{a9(q-@=4)R*BzMkh5RFG8i=N*xZ5zNO=UOYc1*$(+)fPFUxNt+WeRw-qo|mNc Cssz6P literal 0 HcmV?d00001 diff --git a/templates/pages/about.html.njk b/templates/pages/about.html.njk index 4f288e9..ae4ca6a 100644 --- a/templates/pages/about.html.njk +++ b/templates/pages/about.html.njk @@ -37,6 +37,56 @@ -{% endblock %} + {% macro teamMember(contents, speaker) %} +
+ {% if speaker.links.twitter %} + + {% endif %} + {% include '../partials/speaker-picture.html.njk' %} +

+ {{ speaker.firstname }} {{ speaker.lastname }} +

+

+ {% if speaker.twitterHandle %} + {{ speaker.twitterHandle }}
+ {% endif %} +

+ {% if speaker.links.twitter %} + + {% endif %} +
+ {% endmacro %} + + {% set team = contents.team._.pages %} + +
+
+

Organizing team

+
+
+
+ {% for page in team %} + {% set speaker = page.metadata.speaker %} + {% if speaker.core %} + {{ teamMember(contents, speaker) }} + {% endif %} + {% endfor %} +
+
+
+

Volunteer team

+
+
+ +
+ {% for page in team %} + {% set speaker = page.metadata.speaker %} + {% if not speaker.core %} + {{ teamMember(contents, speaker) }} + {% endif %} + {% endfor %} +
+ +{% endblock %} diff --git a/templates/partials/speaker-picture.html.njk b/templates/partials/speaker-picture.html.njk new file mode 100644 index 0000000..b7d914e --- /dev/null +++ b/templates/partials/speaker-picture.html.njk @@ -0,0 +1,11 @@ +{% if speaker.image.filename %} +
+ Portrait photo of {{ speaker.firstname }} {{ speaker.lastname }} +
+{% else %} +
+ +
+{% endif %} From 22ff37141d8e6a7de05e19420db1d2bb28193df7 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 19 Nov 2018 20:21:21 -0800 Subject: [PATCH 2/3] Update file download logic to support checked in copies. In the past image URLs have proven to be pretty unstable. With this change images can be checked so that they aren't repeatedly downloaded if already available in the repo. Also unifies download logic between sponsors and other images that was duplicated for no very good reason. --- contents/images/cms/README.md | 1 + .../image-utils/image-download.js | 98 +++++++++++++++++++ .../image-utils/speaker-image.js | 84 ---------------- .../image-utils/sponsor-image.js | 61 ------------ scripts/spreadsheet-import/index.js | 31 ++---- templates/partials/speaker-picture.html.njk | 2 +- 6 files changed, 110 insertions(+), 167 deletions(-) create mode 100644 contents/images/cms/README.md create mode 100644 scripts/spreadsheet-import/image-utils/image-download.js delete mode 100644 scripts/spreadsheet-import/image-utils/speaker-image.js delete mode 100644 scripts/spreadsheet-import/image-utils/sponsor-image.js diff --git a/contents/images/cms/README.md b/contents/images/cms/README.md new file mode 100644 index 0000000..791420a --- /dev/null +++ b/contents/images/cms/README.md @@ -0,0 +1 @@ +# Make this exist \ No newline at end of file diff --git a/scripts/spreadsheet-import/image-utils/image-download.js b/scripts/spreadsheet-import/image-utils/image-download.js new file mode 100644 index 0000000..6153e0b --- /dev/null +++ b/scripts/spreadsheet-import/image-utils/image-download.js @@ -0,0 +1,98 @@ +const fs = require('fs'); +const path = require('path'); +const fetch = require('node-fetch'); +const chalk = require('chalk'); +const imageType = require('image-type'); +const imageSize = require('image-size'); +const {promisify} = require('util'); + +function getImageFilename(originalUrl, name, ext) { + let filename = name || 'image'; + filename = filename.replace(/[^\w]/g, '-'); + filename = filename.replace(/--/g, '-'); + // Filename changes if underlying URL changes. + const hash = require('crypto').createHash('sha1') + .update(originalUrl).digest('hex'); + filename += '-' + hash.substr(0, 8); + return (filename + '.' + ext).toLowerCase(); +} + +async function existingImage(url, name, opt_extension) { + const extensions = ['jpg', 'png', 'gif', 'jpeg']; + if (opt_extension) { + extensions.unshift(opt_extension); + } + for (let ext of extensions) { + let filename = getImageFilename(url, name, ext); + if (await promisify(fs.exists)(fullPath(filename))) { + console.info(' --> existing image', chalk.green(fullPath(filename))); + return { + ext, + filename, + buffer: await promisify(fs.readFile)(fullPath(filename)), + }; + } + } + return null; +} + +function fullPath(filename) { + return 'contents/images/cms/' + filename; +} + +// Downloads an image from a url unless a local copy is available. +// name should be any string for use in the filename +// Pass in opt_extension if you know the type of the image. +async function downloadImage(url, name, opt_extension) { + console.log('Downloading', url); + if (!url) { + console.error(chalk.yellow('Skipping empty image url')); + return {}; + } + + try { + let info = await existingImage(url, name, opt_extension); + if (!info) { + info = {}; + const res = await fetch(url); + const contentType = res.headers.get('content-type'); + if (!contentType || !contentType.startsWith('image')) { + console.error(chalk.red.bold(' !!! url is not an image', url)); + return {}; + } + + const buffer = await res.buffer(); + info.buffer = buffer; + + const type = opt_extension == 'svg' ? {ext: 'svg'} : imageType(buffer); + if (!type) { + console.error(chalk.red.bold(' !!! no type-information for image', url)); + return {}; + } + info.ext = type.ext; + const filename = getImageFilename(url, name, info.ext); + info.filename = filename; + + const path = fullPath(filename); + console.info(' --> image downloaded ', chalk.green(path)); + fs.writeFileSync(path, buffer); + } + let size = {}; + try { + size = imageSize(info.buffer) || {}; + } catch (e) { + console.error(chalk.yellow('Can\'t get image size' + e.message)); + } + return { + filename: info.filename, + width: size.width, + height: size.height + }; + } catch (err) { + console.error(chalk.red.bold(' !!! failed to download', url)); + console.error(err); + return {}; + } +} + +module.exports = {downloadImage}; diff --git a/scripts/spreadsheet-import/image-utils/speaker-image.js b/scripts/spreadsheet-import/image-utils/speaker-image.js deleted file mode 100644 index c8675c2..0000000 --- a/scripts/spreadsheet-import/image-utils/speaker-image.js +++ /dev/null @@ -1,84 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const fetch = require('node-fetch'); -const chalk = require('chalk'); -const imageType = require('image-type'); -const imageSize = require('image-size'); - -function getImageFilename(speaker, ext) { - let filename = speaker.firstname + '-' + speaker.lastname; - filename = filename.replace(/[^\w]/g, '-'); - filename = filename.replace(/--/g, '-').toLowerCase(); - - return filename + '.' + ext; -} - -function getLocalSpeakerImage(imagePath, speaker) { - if (!imagePath) { - return null; - } - - const filename = getImageFilename(speaker, 'jpg'); - const srcFilename = path.join(imagePath, filename); - const destFilename = path.join('contents/images/speaker', filename); - - if (fs.existsSync(srcFilename)) { - console.log(` --> image found in image-path:`, filename); - const buffer = fs.readFileSync(srcFilename); - const size = imageSize(buffer); - fs.writeFileSync(destFilename, buffer); - - return { - filename, - width: size.width, - height: size.height - }; - } - - return null; -} - -async function downloadSpeakerImage(speaker) { - const url = speaker.potraitImageUrl; - console.log('downloadImage', url); - if (!url) { - console.error(chalk.yellow('no image specified for ' + speaker.id)); - return {}; - } - - try { - const res = await fetch(url); - - if (!res.headers.get('content-type').startsWith('image')) { - console.error(chalk.red.bold(' !!! url is not an image', url)); - return {}; - } - - const buffer = await res.buffer(); - - const info = imageType(buffer); - if (!info) { - console.error(chalk.red.bold(' !!! no type-imformation for image', url)); - return {}; - } - - const size = imageSize(buffer); - const filename = getImageFilename(speaker, info.ext); - const fullPath = 'contents/images/speaker/' + filename; - - console.info(' --> image downloaded ', chalk.green(fullPath)); - fs.writeFileSync(fullPath, buffer); - - return { - filename, - width: size.width, - height: size.height - }; - } catch (err) { - console.error(chalk.red.bold(' !!! failed to download', url)); - console.error(err); - return {}; - } -} - -module.exports = {getLocalSpeakerImage, downloadSpeakerImage}; diff --git a/scripts/spreadsheet-import/image-utils/sponsor-image.js b/scripts/spreadsheet-import/image-utils/sponsor-image.js deleted file mode 100644 index 8926668..0000000 --- a/scripts/spreadsheet-import/image-utils/sponsor-image.js +++ /dev/null @@ -1,61 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const fetch = require('node-fetch'); -const chalk = require('chalk'); - -function getLocalSponsorImage(imagePath, sponsor) { - if (!imagePath) { - return null; - } - - const filename = sponsor.id + '.svg'; - const srcFilename = path.join(imagePath, filename); - const destFilename = path.join('contents/images/sponsor', filename); - - if (fs.existsSync(srcFilename)) { - console.log(` --> image found in image-path:`, filename); - const buffer = fs.readFileSync(srcFilename); - fs.writeFileSync(destFilename, buffer); - - return { - filename - }; - } - - return null; -} - -async function downloadSponsorImage(sponsor) { - const url = sponsor.logoUrl; - console.log('downloadImage', url); - if (!url) { - console.error(chalk.yellow('no image specified for ' + sponsor.id)); - return {}; - } - - try { - const res = await fetch(url); - - if (!res.headers.get('content-type').startsWith('image')) { - console.error(chalk.red.bold(' !!! url is not an image', url)); - return {}; - } - - const buffer = await res.buffer(); - const filename = sponsor.id + '.svg'; - const fullPath = `contents/images/sponsor/${filename}`; - - console.info(' --> image downloaded ', chalk.green(fullPath)); - fs.writeFileSync(fullPath, buffer); - - return { - filename - }; - } catch (err) { - console.error(chalk.red.bold(' !!! failed to download', url)); - console.error(err); - return {}; - } -} - -module.exports = {getLocalSponsorImage, downloadSponsorImage}; diff --git a/scripts/spreadsheet-import/index.js b/scripts/spreadsheet-import/index.js index 64e2dcf..82f435e 100644 --- a/scripts/spreadsheet-import/index.js +++ b/scripts/spreadsheet-import/index.js @@ -8,8 +8,7 @@ const mkdirp = require('mkdirp'); const {promisify} = require('util'); const {getSheetData} = require('./spreadsheet-api'); const {processSheet, simplifySpreadsheetData} = require('./spreadsheet-utils'); -const {downloadSpeakerImage, getLocalSpeakerImage} = require('./image-utils/speaker-image'); -const {downloadSponsorImage, getLocalSponsorImage} = require('./image-utils/sponsor-image'); +const {downloadImage} = require('./image-utils/image-download'); const {processSchedule} = require('./process-schedule'); const rimraf = promisify(require('rimraf')); const timeout = promisify(setTimeout); @@ -193,7 +192,6 @@ async function main(params) { const filename = path.join(contentRoot, contentPath, `${record.id}.md`); let {content, ...data} = record; - let title = ''; if (!content) { content = ' '; @@ -203,25 +201,16 @@ async function main(params) { data.name = data.firstname + ' ' + data.lastname; } + let imageExtension = null; if (sheetId === 'sponsors') { - data.image = getLocalSponsorImage(params.imagePath, data); - title = data.name; - if (!data.image) { - try { - data.image = await downloadSponsorImage(data); - } catch (err) { - console.error('this is bad: ', err); - } - } - delete data.logoUrl; - } else { - data.image = getLocalSpeakerImage(params.imagePath, data); - title = `${data.name}: ${data.talkTitle}`; - if (!data.image) { - data.image = await downloadSpeakerImage(data); - } - - delete data.potraitImageUrl; + imageExtension = 'svg'; + } + const imageUrl = data.potraitImageUrl || data.logoUrl; + data.image = await downloadImage(imageUrl, data.name, imageExtension); + + let title = data.name; + if (data.talkTitle) { + title += `: ${data.talkTitle}`; } const frontmatter = yaml.safeDump({ diff --git a/templates/partials/speaker-picture.html.njk b/templates/partials/speaker-picture.html.njk index b7d914e..7797947 100644 --- a/templates/partials/speaker-picture.html.njk +++ b/templates/partials/speaker-picture.html.njk @@ -1,6 +1,6 @@ {% if speaker.image.filename %}
- Portrait photo of {{ speaker.firstname }} {{ speaker.lastname }}
From 65a447afbddc6c87636dee7b760443c3d2f1c90f Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Tue, 20 Nov 2018 19:25:52 -0800 Subject: [PATCH 3/3] Less sync writing. --- scripts/spreadsheet-import/image-utils/image-download.js | 2 +- scripts/spreadsheet-import/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/spreadsheet-import/image-utils/image-download.js b/scripts/spreadsheet-import/image-utils/image-download.js index 6153e0b..25f26aa 100644 --- a/scripts/spreadsheet-import/image-utils/image-download.js +++ b/scripts/spreadsheet-import/image-utils/image-download.js @@ -75,7 +75,7 @@ async function downloadImage(url, name, opt_extension) { const path = fullPath(filename); console.info(' --> image downloaded ', chalk.green(path)); - fs.writeFileSync(path, buffer); + fs.writeFile(path, buffer, () => {/*fire and forget*/}); } let size = {}; try { diff --git a/scripts/spreadsheet-import/index.js b/scripts/spreadsheet-import/index.js index 82f435e..9f8a3ab 100644 --- a/scripts/spreadsheet-import/index.js +++ b/scripts/spreadsheet-import/index.js @@ -232,7 +232,7 @@ async function main(params) { '\n\n----\n\n' + wordwrap(content); - fs.writeFileSync(filename, markdownContent); + fs.writeFile(filename, markdownContent, () => {/*fire and forget*/}); }); }); }