From b04b1be68fb4f482ea327ac78c82bb4c1b70a37e Mon Sep 17 00:00:00 2001 From: Carmelo Fiorello Date: Tue, 9 May 2023 20:08:44 +0200 Subject: [PATCH 1/4] Add sitemapPath optional argument to scanner interface --- packages/cli/src/ci.ts | 1 + packages/cli/src/createCli.ts | 1 + packages/cli/src/types.ts | 1 + packages/cli/src/util.ts | 3 +++ packages/core/src/discovery/routes.ts | 2 +- packages/core/src/discovery/sitemap.ts | 4 ++-- packages/core/src/types.ts | 6 ++++++ 7 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/ci.ts b/packages/cli/src/ci.ts index 835b1ed4..3979679c 100644 --- a/packages/cli/src/ci.ts +++ b/packages/cli/src/ci.ts @@ -14,6 +14,7 @@ async function run() { cli.option('--budget ', 'Budget (1-100), the minimum score which can pass.') cli.option('--build-static ', 'Build a static website for the reports which can be uploaded.') + cli.option('--sitemap-path ', 'Set a custom path for the sitemap.') const { options } = cli.parse() as unknown as { options: CiOptions } diff --git a/packages/cli/src/createCli.ts b/packages/cli/src/createCli.ts index 82cf3b85..c9cad2f1 100644 --- a/packages/cli/src/createCli.ts +++ b/packages/cli/src/createCli.ts @@ -20,6 +20,7 @@ export default function createCli() { cli.option('--mobile', 'Simulate device as mobile.') cli.option('--site ', 'Host URL to scan') + cli.option('--sitemap-path ', 'Sitemap path to use for scanning.') cli.option('--samples ', 'Specify the amount of samples to run.') cli.option('--throttle', 'Enable the throttling') cli.option('--enable-javascript', 'When inspecting the HTML wait for the javascript to execute. Useful for SPAs.') diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts index 069ba114..4aaacc45 100644 --- a/packages/cli/src/types.ts +++ b/packages/cli/src/types.ts @@ -17,6 +17,7 @@ export interface CliOptions { disableI18nPages?: boolean enableJavascript?: boolean disableJavascript?: boolean + sitemapPath?: string } export interface CiOptions extends CliOptions { diff --git a/packages/cli/src/util.ts b/packages/cli/src/util.ts index 61ceef48..ee971f9c 100644 --- a/packages/cli/src/util.ts +++ b/packages/cli/src/util.ts @@ -66,6 +66,9 @@ export function pickOptions(options: CiOptions | CliOptions): UserConfig { if (options.enableJavascript) picked.scanner.skipJavascript = false + if (options.sitemapPath) + picked.scanner.sitemapPath = options.sitemapPath + else if (options.disableJavascript) picked.scanner.skipJavascript = true diff --git a/packages/core/src/discovery/routes.ts b/packages/core/src/discovery/routes.ts index d444aed5..17bd263a 100644 --- a/packages/core/src/discovery/routes.ts +++ b/packages/core/src/discovery/routes.ts @@ -36,7 +36,7 @@ export const resolveReportableRoutes: () => Promise = async ( // if sitemap scanning is enabled if (resolvedConfig.scanner.sitemap) { - const sitemapUrls = await extractSitemapRoutes(resolvedConfig.site) + const sitemapUrls = await extractSitemapRoutes(resolvedConfig.site, resolvedConfig.scanner.sitemapPath) if (sitemapUrls.length) { logger.info(`Discovered ${sitemapUrls.length} routes from sitemap.xml.`) sitemapUrls.forEach(url => urls.add(url)) diff --git a/packages/core/src/discovery/sitemap.ts b/packages/core/src/discovery/sitemap.ts index b147db17..ee80fd82 100644 --- a/packages/core/src/discovery/sitemap.ts +++ b/packages/core/src/discovery/sitemap.ts @@ -8,7 +8,7 @@ import { useLogger } from '../logger' * * @param site */ -export const extractSitemapRoutes = async (site: string) => { +export const extractSitemapRoutes = async (site: string, sitemapPath?: string) => { // make sure we're working from the host name site = new $URL(site).origin const unlighthouse = useUnlighthouse() @@ -18,7 +18,7 @@ export const extractSitemapRoutes = async (site: string) => { debug: unlighthouse.resolvedConfig.debug, }) - const sitemapUrl = `${site}/sitemap.xml` + const sitemapUrl = `${site}/${sitemapPath || 'sitemap.xml'}` logger.debug(`Attempting to fetch sitemap at ${sitemapUrl}`) const { sites } = await sitemap.fetch(sitemapUrl) logger.debug(`Fetched sitemap with ${sites.length} URLs.`) diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 757bef2e..453e717d 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -399,6 +399,12 @@ export interface ResolvedUserConfig { * @default true */ sitemap: boolean + /** + * Path where get the sitemap. + * + * @default undefined + */ + sitemapPath?: string /** * Alias to switch the device used for scanning. * Set to false if you want to manually configure it. From 944112921c9605b954891e700d5ef4908c2e101a Mon Sep 17 00:00:00 2001 From: Carmelo Fiorello Date: Tue, 9 May 2023 23:18:12 +0200 Subject: [PATCH 2/4] Check if sitemapPath starts with "/" --- packages/core/src/discovery/sitemap.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/core/src/discovery/sitemap.ts b/packages/core/src/discovery/sitemap.ts index ee80fd82..49b73d4b 100644 --- a/packages/core/src/discovery/sitemap.ts +++ b/packages/core/src/discovery/sitemap.ts @@ -18,7 +18,13 @@ export const extractSitemapRoutes = async (site: string, sitemapPath?: string) = debug: unlighthouse.resolvedConfig.debug, }) - const sitemapUrl = `${site}/${sitemapPath || 'sitemap.xml'}` + let sitemapUrl = `${site}/sitemap.xml` + // if sitemapPath exists, check if start with "/" and remove it + if (sitemapPath) { + sitemapPath = sitemapPath.startsWith('/') ? sitemapPath.slice(1) : sitemapPath + sitemapUrl = `${site}/${sitemapPath}` + } + logger.debug(`Attempting to fetch sitemap at ${sitemapUrl}`) const { sites } = await sitemap.fetch(sitemapUrl) logger.debug(`Fetched sitemap with ${sites.length} URLs.`) From 66d0e9955b3e66c25e879bf8580b53bbb7c90d58 Mon Sep 17 00:00:00 2001 From: Carmelo Fiorello Date: Tue, 9 May 2023 23:55:24 +0200 Subject: [PATCH 3/4] Add test unitedpets --- test/fixtures/unitedpets.config.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/fixtures/unitedpets.config.ts diff --git a/test/fixtures/unitedpets.config.ts b/test/fixtures/unitedpets.config.ts new file mode 100644 index 00000000..f056f35f --- /dev/null +++ b/test/fixtures/unitedpets.config.ts @@ -0,0 +1,19 @@ +export default { + site: 'https://unitedpets.com', + outputPath: 'unitedpets-uci', + ci: { + // budget: { + // performance: 50, + // accessibility: 100, + // 'best-practices': 90, + // seo: 90, + // }, + buildStatic: true + }, + scanner: { + sitemapPath: 'sitemap/index.xml', + maxRoutes: 50, + device: 'desktop' + }, + debug: true +} From c037874a13d9ee079aa2e4900b6b0ac20d01159a Mon Sep 17 00:00:00 2001 From: Carmelo Fiorello Date: Wed, 10 May 2023 00:40:36 +0200 Subject: [PATCH 4/4] add routerPrefix to up test --- test/fixtures/unitedpets.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/fixtures/unitedpets.config.ts b/test/fixtures/unitedpets.config.ts index f056f35f..322f2e58 100644 --- a/test/fixtures/unitedpets.config.ts +++ b/test/fixtures/unitedpets.config.ts @@ -1,6 +1,7 @@ export default { site: 'https://unitedpets.com', outputPath: 'unitedpets-uci', + routerPrefix: '/skin-unlighthouse', ci: { // budget: { // performance: 50,