diff --git a/src/DataSource/HostedWordPressApi.js b/src/DataSource/HostedWordPressApi.js index cec0d95..152abbd 100644 --- a/src/DataSource/HostedWordPressApi.js +++ b/src/DataSource/HostedWordPressApi.js @@ -2,7 +2,7 @@ import { DataSource } from "../DataSource.js"; class HostedWordPressApi extends DataSource { static TYPE = "wordpressapi-hosted"; - static TYPE_FRIENDLY = "Hosted WordPress"; + static TYPE_FRIENDLY = "WordPress.com"; static #getHostname(url) { try { diff --git a/src/Fetcher.js b/src/Fetcher.js index 8312fa9..85f0259 100644 --- a/src/Fetcher.js +++ b/src/Fetcher.js @@ -22,7 +22,7 @@ const xmlParser = new XMLParser({ class Fetcher { static USER_AGENT = "Eleventy Import v1.0.0"; - static getFilenameFromSrc(src, fileExtensionFallback) { + static getFilenameFromSrc(src, contentType = "") { let {pathname} = new URL(src); let hash = this.createHash(src); @@ -35,6 +35,7 @@ class Fetcher { return `${filenameWithoutExtension}-${hash}.${extension}`; } + let [, fileExtensionFallback] = contentType.split("/"); // No known file extension return `${filename.slice(0, MAXIMUM_URL_FILENAME_SIZE)}-${hash}${fileExtensionFallback ? `.${fileExtensionFallback}` : ""}`; } @@ -89,67 +90,77 @@ class Fetcher { async fetchAsset(url, outputFolder, urlPath = "assets") { // TODO move this upstream as a Fetch `alias` feature. - let result = await this.fetch(url, { + return this.fetch(url, { type: "buffer", returnType: "response", }, { verbose: true, showErrors: true - }); + }).then(result => { + let filename = Fetcher.getFilenameFromSrc(url, result.headers?.["content-type"]); + let assetUrlLocation = path.join(urlPath, filename); + let fullOutputLocation = path.join(outputFolder, assetUrlLocation); + let urlValue = `/${assetUrlLocation}`; + + if(this.writtenAssetFiles.has(fullOutputLocation)) { + return urlValue; + } - let [, extension] = result.headers?.["content-type"]?.split("/"); - let filename = Fetcher.getFilenameFromSrc(url, extension); - let assetUrlLocation = path.join(urlPath, filename); - let fullOutputLocation = path.join(outputFolder, assetUrlLocation); - let urlValue = `/${assetUrlLocation}`; + this.writtenAssetFiles.add(fullOutputLocation); - if(this.writtenAssetFiles.has(fullOutputLocation)) { - return urlValue; - } + if(this.safeMode && fs.existsSync(fullOutputLocation)) { + if(this.isVerbose) { + Logger.skipping("asset", fullOutputLocation, url); + } + return urlValue; + } - this.writtenAssetFiles.add(fullOutputLocation); + if(this.#directoryManager) { + this.#directoryManager.createDirectoryForPath(fullOutputLocation); + } - if(this.safeMode && fs.existsSync(fullOutputLocation)) { if(this.isVerbose) { - Logger.skipping("asset", fullOutputLocation, url); + Logger.importing("asset", fullOutputLocation, url, { + size: result.body.length, + dryRun: this.dryRun + }); } - return urlValue; - } - if(this.#directoryManager) { - this.#directoryManager.createDirectoryForPath(fullOutputLocation); - } + if(!this.dryRun) { + this.counts.assets++; - if(this.isVerbose) { - Logger.importing("asset", fullOutputLocation, url, { - size: result.body.length, - dryRun: this.dryRun - }); - } - - if(!this.dryRun) { - this.counts.assets++; - - fs.writeFileSync(fullOutputLocation, result.body); - } + fs.writeFileSync(fullOutputLocation, result.body); + } - return urlValue; + return urlValue; + }, error => { + // Logging the error happens in .fetch() upstream + // Fetching the asset failed but we don’t want to fail the upstream document promise + return ""; + }); } async fetch(url, options = {}, verbosity = {}) { let { verbose, showErrors } = Object.assign({ - showErrors: true, - verbose: true, // whether to log the fetch request + verbose: true, // whether to log the initial fetch request + showErrors: true, // whether to show if a request has an error. }, verbosity); let opts = Object.assign({ duration: this.#cacheDuration, type: "text", - verbose: false, // we’re handling our own logging here + verbose: false, // don’t use Fetch logging—we’re handling it ourself fetchOptions: {}, }, options); + if(!opts.fetchOptions.headers) { + opts.fetchOptions.headers = {}; + } + Object.assign(opts.fetchOptions.headers, { + "user-agent": Fetcher.USER_AGENT + }); + if(!this.fetchedUrls.has(url) && this.isVerbose && verbose) { let logAdds = []; if(Boolean(options?.fetchOptions?.headers?.Authorization)) { @@ -164,31 +175,21 @@ class Fetcher { this.fetchedUrls.add(url); - if(!opts.fetchOptions.headers) { - opts.fetchOptions.headers = {}; - } - Object.assign(opts.fetchOptions.headers, { - "user-agent": Fetcher.USER_AGENT - }); - - try { - let result = await EleventyFetch(url, opts); - + return EleventyFetch(url, opts).then(result => { if(opts.type === "xml") { return xmlParser.parse(result); } return result; - } catch(e) { + }, error => { this.errors.add(url); - // if(this.isVerbose && showErrors) { - if(showErrors) { - Logger.log(kleur.red(`Error fetching`), url, kleur.red(e.message)); + if(this.isVerbose && showErrors) { + Logger.log(kleur.red(`Error fetching`), url, kleur.red(error.message)); } - return Promise.reject(e); - } + return Promise.reject(error); + }); } } diff --git a/src/Importer.js b/src/Importer.js index a5118f3..269b5df 100644 --- a/src/Importer.js +++ b/src/Importer.js @@ -169,7 +169,7 @@ class Importer { entries = entries.slice(0, MAX_IMPORT_SIZE); } - let promises = await Promise.all(entries.map(async entry => { + let promises = await Promise.allSettled(entries.map(async entry => { if(Importer.isHtml(entry)) { entry.content = await this.htmlTransformer.transform(entry.content, entry.url); @@ -185,7 +185,11 @@ class Importer { return entry; })); - return promises.sort((a, b) => { + return promises.filter(entry => { + return entry.status !== "rejected"; + }).map(entry => { + return entry.value; + }).sort((a, b) => { if(a.date < b.date) { return 1; } diff --git a/src/Logger.js b/src/Logger.js index 5dc08b0..5678eaa 100644 --- a/src/Logger.js +++ b/src/Logger.js @@ -10,20 +10,23 @@ class Logger { let { size, dryRun } = options; let extras = []; + let prefix = ""; if(label === "Skipping") { - extras.push("overwrites disabled"); + prefix = " (no --overwrite)"; } else { if(size) { extras.push(filesize(size, { spacer: "" })); } + if(dryRun) { - extras.push("dry run"); + prefix = " (dry run)"; } } + let extrasStr = extras.length ? `(${extras.join(", ")}) ` : ""; - this.log(kleur.gray(`${label} ${type}`), local, kleur.gray(`${extrasStr}from`), remote); + this.log(kleur.gray(`${label} ${type}${prefix}`), local, kleur.gray(`${extrasStr}from`), remote); } static importing(type, local, remote, options = {}) {