chore: fix robots tests

harlan-zw · Sep 29, 2024 · c34d06c · c34d06c
1 parent 722522d
commit c34d06c
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/packages/core/src/discovery/robotsTxt.ts b/packages/core/src/discovery/robotsTxt.ts
@@ -105,7 +105,7 @@ export function matchPathToRule(path: string, _rules: RobotsTxtRule[]): RobotsTx
 export function mergeRobotsTxtConfig(config: ResolvedUserConfig, { groups, sitemaps }: RobotsTxtParsed): void {
  config.scanner._robotsTxtRules = groups.filter((group) => {
  return group.userAgent.includes('*') || group.userAgent.includes(String(config.lighthouseOptions?.emulatedUserAgent))
- }).map(group => group._rules)
+ }).flatMap(group => group._rules)
  if (config.scanner.sitemap !== false && sitemaps.length) {
  // allow overriding the robots.txt sitemaps with your own
  if (!Array.isArray(config.scanner.sitemap) || !config.scanner.sitemap.length)

diff --git a/packages/core/src/puppeteer/worker.ts b/packages/core/src/puppeteer/worker.ts
@@ -96,7 +96,7 @@ export async function createUnlighthouseWorker(tasks: Record<UnlighthouseTask, T
  return
 
  // do robots.txt test
- if (resolvedConfig.scanner.robotsTxt) {
+ if (resolvedConfig.scanner.robotsTxt && resolvedConfig.scanner._robotsTxtRules?.length) {
  const rule = matchPathToRule(path, resolvedConfig.scanner._robotsTxtRules)
  if (rule && !rule.allow) {
  logger.info(`Skipping route based on robots.txt rule \`${rule.pattern}\``, { path })