feat: link crawling support

nitrojs · Mar 30, 2022 · 462398f · 462398f
1 parent b9db0e9
commit 462398f
Show file tree

Hide file tree

Showing 9 changed files with 108 additions and 31 deletions.
diff --git a/playground/public/index.html → playground/app.ts b/playground/public/index.html → playground/app.ts
@@ -1,6 +1,8 @@
-<!DOCTYPE html>
-<html lang="en">
+import { defineEventHandler } from 'h3'
 
+export default defineEventHandler(() => {
+  return `<!DOCTYPE html>
+<html lang="en">
   <head>
     <meta charset="utf-8">
     <link rel="icon" type="image/x-icon" href="/favicon.ico">
@@ -19,4 +21,5 @@ <h1>Welcome to Nitro playground!</h1>
     </ul>
   </body>
 
-</html>
+</html>`
+})
diff --git a/playground/nitro.config.ts b/playground/nitro.config.ts
@@ -1,10 +1,8 @@
 import { defineNitroConfig } from '../src'
 
 export default defineNitroConfig({
+  renderer: './app',
   prerender: {
-    routes: [
-      '/api/test',
-      '/api/cache'
-    ]
+    crawlLinks: true
   }
 })
diff --git a/src/build.ts b/src/build.ts
@@ -9,7 +9,6 @@ import { getRollupConfig } from './rollup/config'
 import { prettyPath, writeFile, isDirectory, replaceAll, serializeTemplate } from './utils'
 import { GLOB_SCAN_PATTERN, scanHandlers } from './scan'
 import type { Nitro } from './types'
-import { createNitro } from './nitro'
 
 export async function prepare (nitro: Nitro) {
   await cleanupDir(nitro.options.output.dir)
@@ -61,28 +60,6 @@ export async function build (nitro: Nitro) {
   return nitro.options.dev ? _watch(nitro) : _build(nitro)
 }
 
-export async function prerender (nitro: Nitro) {
-  // Skip if no prerender routes specified
-  const routes = nitro.options.prerender.routes
-  if (!routes.length) {
-    return
-  }
-  // Build with prerender preset
-  nitro = await createNitro({
-    rootDir: nitro.options.rootDir,
-    preset: 'prerender'
-  })
-  await build(nitro)
-  // Import entry
-  const app = await import(resolve(nitro.options.output.serverDir, 'index.mjs'))
-  for (const route of routes) {
-    const res = await app.localFetch(route)
-    const isJSON = (res.headers.get('content-type') || '').includes('json')
-    const contents = await res.text()
-    await writeFile(join(nitro.options.output.publicDir, route + (isJSON ? '.json' : '.html')), contents, true)
-  }
-}
-
 export async function writeTypes (nitro: Nitro) {
   const routeTypes: Record<string, string[]> = {}
 

diff --git a/src/cli.ts b/src/cli.ts
@@ -3,7 +3,8 @@
 import mri from 'mri'
 import { resolve } from 'pathe'
 import { createNitro } from './nitro'
-import { build, prerender, prepare, copyPublicAssets } from './build'
+import { build, prepare, copyPublicAssets } from './build'
+import { prerender } from './prerender'
 import { createDevServer } from './server/dev'
 
 async function main () {

diff --git a/src/index.ts b/src/index.ts
@@ -4,3 +4,4 @@ export * from './scan'
 export * from './server/dev'
 export * from './options'
 export * from './types'
+export * from './prerender'
diff --git a/src/options.ts b/src/options.ts
@@ -31,6 +31,7 @@ const NitroDefaults: NitroConfig = {
   },
   routes: {},
   prerender: {
+    crawlLinks: false,
     routes: []
   },
   publicDir: 'public',

diff --git a/src/prerender.ts b/src/prerender.ts
@@ -0,0 +1,94 @@
+import { resolve, join } from 'pathe'
+import { createNitro } from './nitro'
+import { build } from './build'
+import type { Nitro } from './types'
+import { writeFile } from './utils'
+
+const CRAWL_EXTENSIONS = new Set(['', '.html', '.json'])
+
+export async function prerender (nitro: Nitro) {
+  // Skip if no prerender routes specified
+  const routes = new Set(nitro.options.prerender.routes)
+  if (nitro.options.prerender.crawlLinks && !routes.size) {
+    routes.add('/')
+  }
+  if (!routes.size) {
+    return
+  }
+  // Build with prerender preset
+  nitro = await createNitro({
+    rootDir: nitro.options.rootDir,
+    preset: 'prerender'
+  })
+  await build(nitro)
+
+  // Import renderer entry
+  const app = await import(resolve(nitro.options.output.serverDir, 'index.mjs'))
+
+  // Start prerendering
+  const generatedRoutes = new Set()
+  const generateRoute = async (route: string) => {
+    const res = await app.localFetch(route)
+    const contents = await res.text()
+
+    const additionalExtension = getExtension(route) ? '' : guessExt(res.headers.get('content-type'))
+    const routeWithIndex = route.endsWith('/') ? route + 'index' : route
+    const fileName = routeWithIndex + additionalExtension
+    await writeFile(join(nitro.options.output.publicDir, fileName), contents, true)
+
+    // Crawl Links
+    if (nitro.options.prerender.crawlLinks && fileName.endsWith('.html')) {
+      const extractedLinks = extractLinks(contents, route)
+        .filter(link =>
+          !generatedRoutes.has(link) &&
+          link.match(/^\//g) &&
+          CRAWL_EXTENSIONS.has(getExtension(link))
+        )
+      for (const route of extractedLinks) { routes.add(route) }
+    }
+  }
+
+  const generateAll = async () => {
+    const results = await Promise.all(Array.from(routes).map(async (route) => {
+      if (generatedRoutes.has(route)) { return false }
+      generatedRoutes.add(route)
+      await generateRoute(route).catch(console.error)
+      return true
+    }))
+    return results.filter(Boolean).length // At least one route generated
+  }
+
+  for (let i = 0; i < 100; i++) {
+    const routesGenerated = await generateAll()
+    if (!routesGenerated) {
+      break
+    }
+  }
+}
+
+const LINK_REGEX = /href=['"]?([^'" >]+)/g
+function extractLinks (html: string, _url: string) {
+  const links: string[] = []
+  for (const match of html.matchAll(LINK_REGEX)) {
+    links.push(match[1])
+  }
+  return links
+}
+
+const EXT_REGEX = /\.[a-z0-9]+$/
+function getExtension (path: string): string {
+  return (path.match(EXT_REGEX) || [])[0] || ''
+}
+
+const ContentTypeToExt = {
+  json: '.json',
+  html: '.html'
+}
+function guessExt (contentType: string = ''): string {
+  for (const pattern in ContentTypeToExt) {
+    if (contentType.includes(pattern)) {
+      return ContentTypeToExt[pattern]
+    }
+  }
+  return ''
+}
diff --git a/src/preset.ts b/src/preset.ts
@@ -1,3 +1,4 @@
+
 import type { NitroPreset } from './types'
 
 export function defineNitroPreset (preset: NitroPreset) {

diff --git a/src/types/nitro.ts b/src/types/nitro.ts
@@ -61,6 +61,7 @@ export interface NitroOptions {
   },
 
   prerender: {
+    crawlLinks: boolean
     routes: string[]
   },