From 4cd53dd97427bfb3d0054bd53cd676c01a0da1e6 Mon Sep 17 00:00:00 2001 From: Florian Dieminger Date: Mon, 19 Aug 2024 15:44:18 +0000 Subject: [PATCH 01/11] wip(syntax-highlight): move prism to the client --- build/syntax-highlight.ts | 79 ++++---------------------------- client/src/document/highlight.ts | 77 +++++++++++++++++++++++++++++++ client/src/document/hooks.ts | 7 ++- 3 files changed, 93 insertions(+), 70 deletions(-) create mode 100644 client/src/document/highlight.ts diff --git a/build/syntax-highlight.ts b/build/syntax-highlight.ts index b87f4e32c056..c9bace20deea 100644 --- a/build/syntax-highlight.ts +++ b/build/syntax-highlight.ts @@ -1,9 +1,4 @@ -import Prism from "prismjs"; -import loadLanguages from "prismjs/components/index.js"; -import "prism-svelte"; import * as cheerio from "cheerio"; -import { createHmac } from "node:crypto"; -import { SAMPLE_SIGN_KEY } from "../libs/env/index.js"; const lazy = (creator) => { let res; @@ -16,52 +11,6 @@ const lazy = (creator) => { }; }; -const loadAllLanguages = lazy(() => { - // Some languages are always loaded by Prism, so we can omit them here: - // - Markup (atom, html, markup, mathml, rss, ssml, svg, xml) - // - CSS (css) - // - C-like (clike) - // - JavaScript (javascript, js) - loadLanguages([ - "apacheconf", - "bash", - "batch", - "c", - "cpp", - "cs", - "diff", - "django", - "glsl", - "handlebars", - "http", - "ignore", - "ini", - "java", - "json", - "jsx", - "latex", - "less", - "md", - "nginx", - "php", - "powershell", - "pug", - "python", - "regex", - "rust", - "scss", - "sql", - // 'svelte', // Loaded by `prism-svelte` extension - "toml", - "tsx", - "typescript", - "uri", - "wasm", - "webidl", - "yaml", - ]); -}); - // Add things to this list to help make things convenient. Sometimes // there are `
` whose name is not that which
 // Prism expects. It'd be hard to require that content writers
@@ -84,8 +33,6 @@ const IGNORE = new Set(["none", "text", "plain", "unix"]);
  *
  */
 export function syntaxHighlight($: cheerio.CheerioAPI, doc) {
-  loadAllLanguages();
-
   // Our content will be like this: `
` or
   // `
` so we're technically not looking for an exact
   // match. The wildcard would technically match `
`
@@ -110,28 +57,22 @@ export function syntaxHighlight($: cheerio.CheerioAPI, doc) {
       return;
     }
     const code = $pre.text();
-    if (SAMPLE_SIGN_KEY) {
-      const hmac = createHmac("sha256", SAMPLE_SIGN_KEY);
-      hmac.update(name.toLowerCase());
-      hmac.update(code);
-      const signature = hmac.digest("base64");
-      $pre.attr("data-signature", signature);
-    }
     $pre.wrapAll(`
`); if (!$pre.hasClass("hidden")) { $( `
${name}
` ).insertBefore($pre); } - const grammar = Prism.languages[name]; - if (!grammar) { - console.warn( - `Unable to find a Prism grammar for '${name}' found in ${doc.mdn_url}` - ); - return; // bail! - } - const html = Prism.highlight(code, grammar, name); - const $code = $("").html(html); + //const grammar = Prism.languages[name]; + //if (!grammar) { + // console.warn( + // `Unable to find a Prism grammar for '${name}' found in ${doc.mdn_url}` + // ); + // return; // bail! + //} + //const html = Prism.highlight(code, grammar, name); + //const $code = $("").html(html); + const $code = $("").text(code); $pre.empty().append($code); }); diff --git a/client/src/document/highlight.ts b/client/src/document/highlight.ts new file mode 100644 index 000000000000..dac110b8f655 --- /dev/null +++ b/client/src/document/highlight.ts @@ -0,0 +1,77 @@ +import Prism from "prismjs"; + +const LANGS = [ + "apacheconf", + "bash", + "batch", + "c", + "cpp", + "cs", + "diff", + "django", + "glsl", + "handlebars", + "http", + "ignore", + "ini", + "java", + "json", + "jsx", + "latex", + "less", + "md", + "nginx", + "php", + "powershell", + "pug", + "python", + "regex", + "rust", + "scss", + "sql", + // 'svelte', // Loaded by `prism-svelte` extension + "toml", + "tsx", + "typescript", + "uri", + "wasm", + "webidl", + "yaml", +]; +// Add things to this list to help make things convenient. Sometimes +// there are `
` whose name is not that which
+// Prism expects. It'd be hard to require that content writers
+// have to stick to the exact naming conventions that Prism uses
+// because Prism is an implementation detail.
+const ALIASES = new Map([
+  ["sh", "shell"],
+  ["vue", "markup"], // See https://github.com/PrismJS/prism/issues/1665#issuecomment-536529608
+]);
+
+// Over the years we have accumulated some weird 
 tags whose
+// brush is more or less "junk".
+// TODO: Perhaps, if you have a doc with 
 tags that matches
+// this, it should become a flaw.
+
+const IGNORE = new Set(["none", "text", "plain", "unix"]);
+
+export async function highlightSyntax(element: Element, language: string) {
+  const resolvedLanguage = ALIASES.get(language) || language;
+  if (IGNORE.has(resolvedLanguage)) {
+    return;
+  }
+
+  let prismLanguage = Prism.languages[resolvedLanguage];
+  if (!prismLanguage) {
+    if (LANGS.includes(resolvedLanguage)) {
+      await import(`prismjs/components/prism-${resolvedLanguage}.js`);
+    } else if (resolvedLanguage === "svelte") {
+      await import("prism-svelte");
+    }
+  }
+  prismLanguage = Prism.languages[resolvedLanguage];
+
+  if (prismLanguage) {
+    element.innerHTML = Prism.highlight(element.textContent, prismLanguage);
+  }
+}
diff --git a/client/src/document/hooks.ts b/client/src/document/hooks.ts
index 38c9114cea10..0a7d3e463883 100644
--- a/client/src/document/hooks.ts
+++ b/client/src/document/hooks.ts
@@ -13,6 +13,7 @@ import {
 } from "./code/playground";
 import { addCopyToClipboardButton } from "./code/copy";
 import { useUIStatus } from "../ui-context";
+import { highlightSyntax } from "./highlight";
 
 export function useDocumentURL() {
   const locale = useLocale();
@@ -110,7 +111,7 @@ export function useCopyExamplesToClipboardAndAIExplain(doc: Doc | undefined) {
 
     document
       .querySelectorAll("div.code-example pre:not(.hidden)")
-      .forEach((element) => {
+      .forEach(async (element) => {
         const header = element.parentElement?.querySelector(".example-header");
         // Paused for now
         // addExplainButton(header, element);
@@ -120,6 +121,10 @@ export function useCopyExamplesToClipboardAndAIExplain(doc: Doc | undefined) {
           );
           return;
         } else {
+          await highlightSyntax(
+            element,
+            header?.querySelector(".language-name")?.textContent || "plain"
+          );
           addCopyToClipboardButton(element, header);
         }
       });

From 1629d271737c75d72d571d50b43b76472338f267 Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Mon, 19 Aug 2024 15:46:09 +0000
Subject: [PATCH 02/11] wip(syntax-highlight): types, no blocking, no
 allowlist, wrap in 

---
 client/src/document/highlight.ts | 58 ++++++++------------------------
 client/src/document/hooks.ts     | 10 +++---
 package.json                     |  1 +
 yarn.lock                        |  5 +++
 4 files changed, 25 insertions(+), 49 deletions(-)

diff --git a/client/src/document/highlight.ts b/client/src/document/highlight.ts
index dac110b8f655..92a1e9423e0f 100644
--- a/client/src/document/highlight.ts
+++ b/client/src/document/highlight.ts
@@ -1,43 +1,7 @@
 import Prism from "prismjs";
 
-const LANGS = [
-  "apacheconf",
-  "bash",
-  "batch",
-  "c",
-  "cpp",
-  "cs",
-  "diff",
-  "django",
-  "glsl",
-  "handlebars",
-  "http",
-  "ignore",
-  "ini",
-  "java",
-  "json",
-  "jsx",
-  "latex",
-  "less",
-  "md",
-  "nginx",
-  "php",
-  "powershell",
-  "pug",
-  "python",
-  "regex",
-  "rust",
-  "scss",
-  "sql",
-  // 'svelte', // Loaded by `prism-svelte` extension
-  "toml",
-  "tsx",
-  "typescript",
-  "uri",
-  "wasm",
-  "webidl",
-  "yaml",
-];
+Prism.manual = true;
+
 // Add things to this list to help make things convenient. Sometimes
 // there are `
` whose name is not that which
 // Prism expects. It'd be hard to require that content writers
@@ -52,7 +16,6 @@ const ALIASES = new Map([
 // brush is more or less "junk".
 // TODO: Perhaps, if you have a doc with 
 tags that matches
 // this, it should become a flaw.
-
 const IGNORE = new Set(["none", "text", "plain", "unix"]);
 
 export async function highlightSyntax(element: Element, language: string) {
@@ -63,15 +26,22 @@ export async function highlightSyntax(element: Element, language: string) {
 
   let prismLanguage = Prism.languages[resolvedLanguage];
   if (!prismLanguage) {
-    if (LANGS.includes(resolvedLanguage)) {
-      await import(`prismjs/components/prism-${resolvedLanguage}.js`);
-    } else if (resolvedLanguage === "svelte") {
+    if (resolvedLanguage === "svelte") {
       await import("prism-svelte");
+    } else {
+      try {
+        await import(
+          /* webpackChunkName: "prism" */
+          `prismjs/components/prism-${resolvedLanguage}.js`
+        );
+      } catch {
+        return;
+      }
     }
   }
-  prismLanguage = Prism.languages[resolvedLanguage];
 
+  prismLanguage = Prism.languages[resolvedLanguage];
   if (prismLanguage) {
-    element.innerHTML = Prism.highlight(element.textContent, prismLanguage);
+    element.innerHTML = `${Prism.highlight(element.textContent || "", prismLanguage, resolvedLanguage)}`;
   }
 }
diff --git a/client/src/document/hooks.ts b/client/src/document/hooks.ts
index 0a7d3e463883..33a77e78c1a8 100644
--- a/client/src/document/hooks.ts
+++ b/client/src/document/hooks.ts
@@ -111,7 +111,7 @@ export function useCopyExamplesToClipboardAndAIExplain(doc: Doc | undefined) {
 
     document
       .querySelectorAll("div.code-example pre:not(.hidden)")
-      .forEach(async (element) => {
+      .forEach((element) => {
         const header = element.parentElement?.querySelector(".example-header");
         // Paused for now
         // addExplainButton(header, element);
@@ -121,12 +121,12 @@ export function useCopyExamplesToClipboardAndAIExplain(doc: Doc | undefined) {
           );
           return;
         } else {
-          await highlightSyntax(
-            element,
-            header?.querySelector(".language-name")?.textContent || "plain"
-          );
           addCopyToClipboardButton(element, header);
         }
+        highlightSyntax(
+          element,
+          header?.querySelector(".language-name")?.textContent || "plain"
+        );
       });
   }, [doc, location, isServer]);
 }
diff --git a/package.json b/package.json
index c592d03f0d8b..9e14c3170d43 100644
--- a/package.json
+++ b/package.json
@@ -168,6 +168,7 @@
     "@types/jest": "^29.5.12",
     "@types/mdast": "^4.0.4",
     "@types/node": "^18.19.45",
+    "@types/prismjs": "^1.26.4",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "@types/react-modal": "^3.16.3",
diff --git a/yarn.lock b/yarn.lock
index 2dda4c45a0a6..ae8a2680be5e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3573,6 +3573,11 @@
   resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.1.tgz#dfd20e2dc35f027cdd6c1908e80a5ddc7499670e"
   integrity sha512-ri0UmynRRvZiiUJdiz38MmIblKK+oH30MztdBVR95dv/Ubw6neWSb8u1XpRb72L4qsZOhz+L+z9JD40SJmfWow==
 
+"@types/prismjs@^1.26.4":
+  version "1.26.4"
+  resolved "https://registry.yarnpkg.com/@types/prismjs/-/prismjs-1.26.4.tgz#1a9e1074619ce1d7322669e5b46fbe823925103a"
+  integrity sha512-rlAnzkW2sZOjbqZ743IHUhFcvzaGbqijwOu8QZnZCjfQzBqFE3s4lOTJEsxikImav9uzz/42I+O7YUs1mWgMlg==
+
 "@types/prop-types@*":
   version "15.7.5"
   resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.5.tgz#5f19d2b85a98e9558036f6a3cacc8819420f05cf"

From eae1973f2ef2855d935980095a36d0dc5f937c03 Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Mon, 19 Aug 2024 15:59:19 +0000
Subject: [PATCH 03/11] wip(syntax-highlight): fix prism types in ai help

---
 client/src/plus/ai-help/index.tsx | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/client/src/plus/ai-help/index.tsx b/client/src/plus/ai-help/index.tsx
index fa089a7ba001..5b52d12e7c5f 100644
--- a/client/src/plus/ai-help/index.tsx
+++ b/client/src/plus/ai-help/index.tsx
@@ -60,6 +60,8 @@ import {
 import InternalLink from "../../ui/atoms/internal-link";
 import { isPlusSubscriber } from "../../utils";
 
+Prism.manual = true;
+
 type Category = "apis" | "css" | "html" | "http" | "js" | "learn";
 
 const EXAMPLES: { category: Category; query: string }[] = [
@@ -482,13 +484,17 @@ function AIHelpAssistantResponse({
               },
               code: ({ className, children, ...props }) => {
                 const match = /language-(\w+)/.exec(className || "");
-                const lang = Prism.languages[match?.[1]];
+                const lang = match?.[1];
                 return lang ? (
                   
                 ) : (

From 350bd45d12c0c177e59252276d08ef06acc569ea Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Mon, 19 Aug 2024 16:19:00 +0000
Subject: [PATCH 04/11] wip(syntax-highlight): rename hook

---
 client/src/blog/post.tsx      |  7 ++-----
 client/src/document/hooks.ts  | 10 +++-------
 client/src/document/index.tsx |  8 ++------
 3 files changed, 7 insertions(+), 18 deletions(-)

diff --git a/client/src/blog/post.tsx b/client/src/blog/post.tsx
index 75282a72fc3d..b02f4e430f07 100644
--- a/client/src/blog/post.tsx
+++ b/client/src/blog/post.tsx
@@ -14,10 +14,7 @@ import {
   BlogPostLimitedMetadata,
   AuthorMetadata,
 } from "../../../libs/types/blog";
-import {
-  useCopyExamplesToClipboardAndAIExplain,
-  useRunSample,
-} from "../document/hooks";
+import { useDecorateExamples, useRunSample } from "../document/hooks";
 import { DEFAULT_LOCALE } from "../../../libs/constants";
 import { SignUpSection as NewsletterSignUp } from "../newsletter";
 import { TOC } from "../document/organisms/toc";
@@ -190,7 +187,7 @@ export function BlogPost(props: HydrationData) {
   );
   const { doc, blogMeta } = data || props || {};
   useRunSample(doc);
-  useCopyExamplesToClipboardAndAIExplain(doc);
+  useDecorateExamples(doc);
   return (
     <>
       {doc && blogMeta && (
diff --git a/client/src/document/hooks.ts b/client/src/document/hooks.ts
index 33a77e78c1a8..6bdd451bec3f 100644
--- a/client/src/document/hooks.ts
+++ b/client/src/document/hooks.ts
@@ -96,15 +96,11 @@ export function useRunSample(doc: Doc | undefined) {
     });
   }, [doc, isServer, locale]);
 }
-export function useCopyExamplesToClipboardAndAIExplain(doc: Doc | undefined) {
+
+export function useDecorateExamples(doc: Doc | undefined) {
   const location = useLocation();
-  const isServer = useIsServer();
 
   useEffect(() => {
-    if (isServer) {
-      return;
-    }
-
     if (!doc) {
       return;
     }
@@ -128,7 +124,7 @@ export function useCopyExamplesToClipboardAndAIExplain(doc: Doc | undefined) {
           header?.querySelector(".language-name")?.textContent || "plain"
         );
       });
-  }, [doc, location, isServer]);
+  }, [doc, location]);
 }
 
 /**
diff --git a/client/src/document/index.tsx b/client/src/document/index.tsx
index bc19fe391833..f67715bfef11 100644
--- a/client/src/document/index.tsx
+++ b/client/src/document/index.tsx
@@ -6,11 +6,7 @@ import { WRITER_MODE, PLACEMENT_ENABLED } from "../env";
 import { useGA } from "../ga-context";
 import { useIsServer, useLocale } from "../hooks";
 
-import {
-  useDocumentURL,
-  useCopyExamplesToClipboardAndAIExplain,
-  useRunSample,
-} from "./hooks";
+import { useDocumentURL, useDecorateExamples, useRunSample } from "./hooks";
 import { Doc } from "../../../libs/types/document";
 // Ingredients
 import { Prose } from "./ingredients/prose";
@@ -124,7 +120,7 @@ export function Document(props /* TODO: define a TS interface for this */) {
   useIncrementFrequentlyViewed(doc);
   useRunSample(doc);
   //useCollectSample(doc);
-  useCopyExamplesToClipboardAndAIExplain(doc);
+  useDecorateExamples(doc);
   useInteractiveExamplesTelemetry();
 
   React.useEffect(() => {

From d5c7eab9ff52ffed8d42ddd1091d53130ba9b205 Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Mon, 19 Aug 2024 16:43:30 +0000
Subject: [PATCH 05/11] wip(syntax-highlight): clean up and rename build step

---
 build/blog.ts                                 |  4 +-
 .../{syntax-highlight.ts => code-headers.ts}  | 42 ++-----------------
 build/curriculum.ts                           |  4 +-
 build/index.ts                                |  6 +--
 client/src/document/highlight.ts              |  9 ----
 5 files changed, 11 insertions(+), 54 deletions(-)
 rename build/{syntax-highlight.ts => code-headers.ts} (50%)

diff --git a/build/blog.ts b/build/blog.ts
index 715115345a98..5a9b4f9b6d9b 100644
--- a/build/blog.ts
+++ b/build/blog.ts
@@ -28,7 +28,7 @@ import {
   postProcessSmallerHeadingIDs,
 } from "./utils.js";
 import { slugToFolder } from "../libs/slug-utils/index.js";
-import { syntaxHighlight } from "./syntax-highlight.js";
+import { codeHeaders } from "./code-headers.js";
 import { wrapTables } from "./wrap-tables.js";
 import { Doc } from "../libs/types/document.js";
 import { extractSections } from "./extract-sections.js";
@@ -391,7 +391,7 @@ export async function buildPost(
     doc.hasMathML = true;
   }
   $("div.hidden").remove();
-  syntaxHighlight($, doc);
+  codeHeaders($);
   injectNoTranslate($);
   injectLoadingLazyAttributes($);
   postProcessExternalLinks($);
diff --git a/build/syntax-highlight.ts b/build/code-headers.ts
similarity index 50%
rename from build/syntax-highlight.ts
rename to build/code-headers.ts
index c9bace20deea..234e5def23dc 100644
--- a/build/syntax-highlight.ts
+++ b/build/code-headers.ts
@@ -1,26 +1,5 @@
 import * as cheerio from "cheerio";
 
-const lazy = (creator) => {
-  let res;
-  let processed = false;
-  return (...args) => {
-    if (processed) return res;
-    res = creator.apply(this, args);
-    processed = true;
-    return res;
-  };
-};
-
-// Add things to this list to help make things convenient. Sometimes
-// there are `
` whose name is not that which
-// Prism expects. It'd be hard to require that content writers
-// have to stick to the exact naming conventions that Prism uses
-// because Prism is an implementation detail.
-const ALIASES = new Map([
-  ["sh", "shell"],
-  ["vue", "markup"], // See https://github.com/PrismJS/prism/issues/1665#issuecomment-536529608
-]);
-
 // Over the years we have accumulated some weird 
 tags whose
 // brush is more or less "junk".
 // TODO: Perhaps, if you have a doc with 
 tags that matches
@@ -28,16 +7,15 @@ const ALIASES = new Map([
 const IGNORE = new Set(["none", "text", "plain", "unix"]);
 
 /**
- * Mutate the `$` instance for by looking for 
 tags that can be
- * syntax highlighted with Prism.
+ * Mutate the `$` instance by adding headers to 
 tags containing code blocks.
  *
  */
-export function syntaxHighlight($: cheerio.CheerioAPI, doc) {
+export function codeHeaders($: cheerio.CheerioAPI) {
   // Our content will be like this: `
` or
   // `
` so we're technically not looking for an exact
   // match. The wildcard would technically match `
`
   // too. But within the loop, we do a more careful regex on the class name
-  // and only proceed if it's something sensible we can use in Prism.
+  // and only proceed if it's something sensible.
   $("pre[class*=brush]").each((_, element) => {
     // The language is whatever string comes after the `brush(:)`
     // portion of the class name.
@@ -48,10 +26,7 @@ export function syntaxHighlight($: cheerio.CheerioAPI, doc) {
     if (!match) {
       return;
     }
-    let name = match[1].replace("-nolint", "");
-    if (ALIASES.has(name)) {
-      name = ALIASES.get(name);
-    }
+    const name = match[1].replace("-nolint", "");
     if (IGNORE.has(name)) {
       // Seems to exist a couple of these in our docs. Just bail.
       return;
@@ -63,15 +38,6 @@ export function syntaxHighlight($: cheerio.CheerioAPI, doc) {
         `
${name}
` ).insertBefore($pre); } - //const grammar = Prism.languages[name]; - //if (!grammar) { - // console.warn( - // `Unable to find a Prism grammar for '${name}' found in ${doc.mdn_url}` - // ); - // return; // bail! - //} - //const html = Prism.highlight(code, grammar, name); - //const $code = $("").html(html); const $code = $("").text(code); $pre.empty().append($code); diff --git a/build/curriculum.ts b/build/curriculum.ts index 863e338070ff..7b4646534adc 100644 --- a/build/curriculum.ts +++ b/build/curriculum.ts @@ -9,7 +9,7 @@ import { DocParent } from "../libs/types/document.js"; import { CURRICULUM_TITLE, DEFAULT_LOCALE } from "../libs/constants/index.js"; import * as kumascript from "../kumascript/index.js"; import LANGUAGES_RAW from "../libs/languages/index.js"; -import { syntaxHighlight } from "./syntax-highlight.js"; +import { codeHeaders } from "./code-headers.js"; import { escapeRegExp, injectLoadingLazyAttributes, @@ -321,7 +321,7 @@ export async function buildCurriculumPage( doc.hasMathML = true; } $("div.hidden").remove(); - syntaxHighlight($, doc); + codeHeaders($); injectNoTranslate($); injectLoadingLazyAttributes($); postProcessCurriculumLinks($, (p: string | undefined) => { diff --git a/build/index.ts b/build/index.ts index 34946c41cc07..87e5cd3d1a20 100644 --- a/build/index.ts +++ b/build/index.ts @@ -28,7 +28,7 @@ import { } from "./flaws/index.js"; import { checkImageReferences, checkImageWidths } from "./check-images.js"; import { getPageTitle } from "./page-title.js"; -import { syntaxHighlight } from "./syntax-highlight.js"; +import { codeHeaders } from "./code-headers.js"; import { formatNotecards } from "./format-notecards.js"; import buildOptions from "./build-options.js"; import LANGUAGES_RAW from "../libs/languages/index.js"; @@ -456,8 +456,8 @@ export async function buildDocument( plainHTML = $.html(); } - // Apply syntax highlighting all
 tags.
-  syntaxHighlight($, doc);
+  // Add headers to all 
 tags with code.
+  codeHeaders($);
 
   // Post process HTML so that the right elements gets tagged so they
   // *don't* get translated by tools like Google Translate.
diff --git a/client/src/document/highlight.ts b/client/src/document/highlight.ts
index 92a1e9423e0f..f94acdca083e 100644
--- a/client/src/document/highlight.ts
+++ b/client/src/document/highlight.ts
@@ -12,17 +12,8 @@ const ALIASES = new Map([
   ["vue", "markup"], // See https://github.com/PrismJS/prism/issues/1665#issuecomment-536529608
 ]);
 
-// Over the years we have accumulated some weird 
 tags whose
-// brush is more or less "junk".
-// TODO: Perhaps, if you have a doc with 
 tags that matches
-// this, it should become a flaw.
-const IGNORE = new Set(["none", "text", "plain", "unix"]);
-
 export async function highlightSyntax(element: Element, language: string) {
   const resolvedLanguage = ALIASES.get(language) || language;
-  if (IGNORE.has(resolvedLanguage)) {
-    return;
-  }
 
   let prismLanguage = Prism.languages[resolvedLanguage];
   if (!prismLanguage) {

From 5943f3493c3e58debe12cedee793f33a7fe256cf Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Fri, 6 Sep 2024 10:27:13 +0000
Subject: [PATCH 06/11] fix(syntax-highlight): handle aliases and dependencies

---
 client/src/document/highlight.ts | 84 +++++++++++++++++++++++++++-----
 1 file changed, 71 insertions(+), 13 deletions(-)

diff --git a/client/src/document/highlight.ts b/client/src/document/highlight.ts
index f94acdca083e..9b886e4b8e03 100644
--- a/client/src/document/highlight.ts
+++ b/client/src/document/highlight.ts
@@ -1,4 +1,15 @@
 import Prism from "prismjs";
+import components from "prismjs/components";
+
+const PRISM_LANGUAGES = components.languages as Record<
+  string,
+  {
+    alias?: string | string[];
+    require?: string | string[];
+    optional?: string | string[];
+    [key: string]: any;
+  }
+>;
 
 Prism.manual = true;
 
@@ -8,31 +19,78 @@ Prism.manual = true;
 // have to stick to the exact naming conventions that Prism uses
 // because Prism is an implementation detail.
 const ALIASES = new Map([
-  ["sh", "shell"],
   ["vue", "markup"], // See https://github.com/PrismJS/prism/issues/1665#issuecomment-536529608
+  ...Object.entries(PRISM_LANGUAGES).flatMap(([lang, config]) => {
+    if (config.alias) {
+      const aliases =
+        typeof config.alias === "string" ? [config.alias] : config.alias;
+      return aliases.map((alias) => [alias, lang] satisfies [string, string]);
+    }
+    return [];
+  }),
 ]);
 
 export async function highlightSyntax(element: Element, language: string) {
   const resolvedLanguage = ALIASES.get(language) || language;
 
-  let prismLanguage = Prism.languages[resolvedLanguage];
+  try {
+    await importLanguage(resolvedLanguage);
+  } catch {
+    return;
+  }
+
+  const prismLanguage = Prism.languages[resolvedLanguage];
+  if (prismLanguage) {
+    element.innerHTML = `${Prism.highlight(element.textContent || "", prismLanguage, resolvedLanguage)}`;
+  }
+}
+
+async function importLanguage(language: string) {
+  const prismLanguage = Prism.languages[language];
+
   if (!prismLanguage) {
-    if (resolvedLanguage === "svelte") {
-      await import("prism-svelte");
+    if (language === "svelte") {
+      try {
+        await import(
+          /* webpackChunkName: "prism-svelte" */
+          "prism-svelte"
+        );
+      } catch (e) {
+        console.warn(`Failed to import ${language} prism language`);
+        throw e;
+      }
     } else {
+      const config = PRISM_LANGUAGES[language];
+      if (config.require) {
+        try {
+          await Promise.all(
+            (typeof config.require === "string"
+              ? [config.require]
+              : config.require
+            ).map((dependency) => importLanguage(dependency))
+          );
+        } catch {
+          return;
+        }
+      }
+      if (config.optional) {
+        await Promise.allSettled(
+          (typeof config.optional === "string"
+            ? [config.optional]
+            : config.optional
+          ).map((dependency) => importLanguage(dependency))
+        );
+      }
       try {
         await import(
-          /* webpackChunkName: "prism" */
-          `prismjs/components/prism-${resolvedLanguage}.js`
+          /* webpackChunkName: "[request]" */
+          /* webpackExclude: /\.min\.js$/ */
+          `prismjs/components/prism-${language}.js`
         );
-      } catch {
-        return;
+      } catch (e) {
+        console.warn(`Failed to import ${language} prism language`);
+        throw e;
       }
     }
   }
-
-  prismLanguage = Prism.languages[resolvedLanguage];
-  if (prismLanguage) {
-    element.innerHTML = `${Prism.highlight(element.textContent || "", prismLanguage, resolvedLanguage)}`;
-  }
 }

From 8b77b12621b17702e28c0e0f7f49be8bbf5a7cc5 Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Fri, 6 Sep 2024 15:43:42 +0000
Subject: [PATCH 07/11] fix(syntax-highlight): add react component for use in
 ai-help

---
 .../syntax-highlight.tsx}                     | 55 +++++++++++++++++--
 client/src/document/hooks.ts                  |  4 +-
 client/src/plus/ai-help/index.tsx             | 24 +++-----
 3 files changed, 59 insertions(+), 24 deletions(-)
 rename client/src/document/{highlight.ts => code/syntax-highlight.tsx} (65%)

diff --git a/client/src/document/highlight.ts b/client/src/document/code/syntax-highlight.tsx
similarity index 65%
rename from client/src/document/highlight.ts
rename to client/src/document/code/syntax-highlight.tsx
index 9b886e4b8e03..554d7c0e45ad 100644
--- a/client/src/document/highlight.ts
+++ b/client/src/document/code/syntax-highlight.tsx
@@ -1,5 +1,8 @@
 import Prism from "prismjs";
 import components from "prismjs/components";
+import { useMemo, useState, useEffect } from "react";
+
+Prism.manual = true;
 
 const PRISM_LANGUAGES = components.languages as Record<
   string,
@@ -11,8 +14,6 @@ const PRISM_LANGUAGES = components.languages as Record<
   }
 >;
 
-Prism.manual = true;
-
 // Add things to this list to help make things convenient. Sometimes
 // there are `
` whose name is not that which
 // Prism expects. It'd be hard to require that content writers
@@ -30,19 +31,63 @@ const ALIASES = new Map([
   }),
 ]);
 
-export async function highlightSyntax(element: Element, language: string) {
+interface HighlightedCodeProps extends React.HTMLAttributes {
+  language?: string;
+  children: React.ReactNode;
+}
+
+export function HighlightedElement({
+  language,
+  children,
+  ...props
+}: HighlightedCodeProps) {
+  const initial = useMemo(
+    // needed to prevent flashing
+    () =>
+      language ? highlightStringSync(String(children), language) : undefined,
+    [children, language]
+  );
+  const [html, setHtml] = useState(initial);
+
+  useEffect(() => {
+    (async () => {
+      if (language) {
+        const highlighted = await highlightString(String(children), language);
+        setHtml(highlighted);
+      }
+    })();
+  }, [children, language]);
+
+  return html ? (
+    
+  ) : (
+    {children}
+  );
+}
+
+export async function highlightElement(element: Element, language: string) {
+  element.innerHTML = `${await highlightString(element.textContent || "", language)}`;
+}
+
+async function highlightString(text: string, language: string) {
   const resolvedLanguage = ALIASES.get(language) || language;
 
   try {
     await importLanguage(resolvedLanguage);
   } catch {
-    return;
+    return text;
   }
 
+  return highlightStringSync(text, language);
+}
+
+function highlightStringSync(text: string, language: string) {
+  const resolvedLanguage = ALIASES.get(language) || language;
   const prismLanguage = Prism.languages[resolvedLanguage];
   if (prismLanguage) {
-    element.innerHTML = `${Prism.highlight(element.textContent || "", prismLanguage, resolvedLanguage)}`;
+    return Prism.highlight(text, prismLanguage, resolvedLanguage);
   }
+  return text;
 }
 
 async function importLanguage(language: string) {
diff --git a/client/src/document/hooks.ts b/client/src/document/hooks.ts
index 6bdd451bec3f..caef0e44c10e 100644
--- a/client/src/document/hooks.ts
+++ b/client/src/document/hooks.ts
@@ -13,7 +13,7 @@ import {
 } from "./code/playground";
 import { addCopyToClipboardButton } from "./code/copy";
 import { useUIStatus } from "../ui-context";
-import { highlightSyntax } from "./highlight";
+import { highlightElement } from "./code/syntax-highlight";
 
 export function useDocumentURL() {
   const locale = useLocale();
@@ -119,7 +119,7 @@ export function useDecorateExamples(doc: Doc | undefined) {
         } else {
           addCopyToClipboardButton(element, header);
         }
-        highlightSyntax(
+        highlightElement(
           element,
           header?.querySelector(".language-name")?.textContent || "plain"
         );
diff --git a/client/src/plus/ai-help/index.tsx b/client/src/plus/ai-help/index.tsx
index 5b52d12e7c5f..fbd2f782621e 100644
--- a/client/src/plus/ai-help/index.tsx
+++ b/client/src/plus/ai-help/index.tsx
@@ -1,4 +1,3 @@
-import Prism from "prismjs";
 import {
   Children,
   MutableRefObject,
@@ -59,8 +58,7 @@ import {
 } from "./constants";
 import InternalLink from "../../ui/atoms/internal-link";
 import { isPlusSubscriber } from "../../utils";
-
-Prism.manual = true;
+import { HighlightedElement } from "../../document/code/syntax-highlight";
 
 type Category = "apis" | "css" | "html" | "http" | "js" | "learn";
 
@@ -485,22 +483,14 @@ function AIHelpAssistantResponse({
               code: ({ className, children, ...props }) => {
                 const match = /language-(\w+)/.exec(className || "");
                 const lang = match?.[1];
-                return lang ? (
-                  
-                ) : (
-                  
+                    {...props}
+                  >
                     {children}
-                  
+                  
                 );
               },
             }}

From 713d8d83f33a28b866eac7ed8c6f26f6e7dbf4e8 Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Mon, 9 Sep 2024 09:21:32 +0000
Subject: [PATCH 08/11] wip(syntax-highlight): import client code async

---
 client/src/document/hooks.ts | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/client/src/document/hooks.ts b/client/src/document/hooks.ts
index caef0e44c10e..e104aab9fa96 100644
--- a/client/src/document/hooks.ts
+++ b/client/src/document/hooks.ts
@@ -13,7 +13,6 @@ import {
 } from "./code/playground";
 import { addCopyToClipboardButton } from "./code/copy";
 import { useUIStatus } from "../ui-context";
-import { highlightElement } from "./code/syntax-highlight";
 
 export function useDocumentURL() {
   const locale = useLocale();
@@ -119,10 +118,12 @@ export function useDecorateExamples(doc: Doc | undefined) {
         } else {
           addCopyToClipboardButton(element, header);
         }
-        highlightElement(
-          element,
-          header?.querySelector(".language-name")?.textContent || "plain"
-        );
+        import("./code/syntax-highlight").then(({ highlightElement }) => {
+          highlightElement(
+            element,
+            header?.querySelector(".language-name")?.textContent || "plain"
+          );
+        });
       });
   }, [doc, location]);
 }

From aa1c5dad9e478db29b8ab69e5e8475c19998685d Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Tue, 10 Sep 2024 12:04:18 +0000
Subject: [PATCH 09/11] fix(syntax-highlight): don't return unescaped strings

---
 client/src/document/code/syntax-highlight.tsx | 22 ++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/client/src/document/code/syntax-highlight.tsx b/client/src/document/code/syntax-highlight.tsx
index 554d7c0e45ad..76acdd32cc51 100644
--- a/client/src/document/code/syntax-highlight.tsx
+++ b/client/src/document/code/syntax-highlight.tsx
@@ -66,28 +66,40 @@ export function HighlightedElement({
 }
 
 export async function highlightElement(element: Element, language: string) {
-  element.innerHTML = `${await highlightString(element.textContent || "", language)}`;
+  const highlighted = await highlightString(
+    element.textContent || "",
+    language
+  );
+  if (highlighted) {
+    element.innerHTML = `${highlighted}`;
+  }
 }
 
-async function highlightString(text: string, language: string) {
+async function highlightString(
+  text: string,
+  language: string
+): Promise {
   const resolvedLanguage = ALIASES.get(language) || language;
 
   try {
     await importLanguage(resolvedLanguage);
   } catch {
-    return text;
+    return;
   }
 
   return highlightStringSync(text, language);
 }
 
-function highlightStringSync(text: string, language: string) {
+function highlightStringSync(
+  text: string,
+  language: string
+): string | undefined {
   const resolvedLanguage = ALIASES.get(language) || language;
   const prismLanguage = Prism.languages[resolvedLanguage];
   if (prismLanguage) {
     return Prism.highlight(text, prismLanguage, resolvedLanguage);
   }
-  return text;
+  return;
 }
 
 async function importLanguage(language: string) {

From ddf0d0f1db71ed691f008804c920bf6a1f7603fa Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Tue, 10 Sep 2024 16:44:49 +0000
Subject: [PATCH 10/11] wip(syntax-highlight): review updates

---
 build/blog.ts                                 | 4 ++--
 build/code-headers.ts                         | 2 +-
 build/curriculum.ts                           | 4 ++--
 build/index.ts                                | 4 ++--
 client/src/blog/post.tsx                      | 4 ++--
 client/src/document/code/syntax-highlight.tsx | 2 +-
 client/src/document/hooks.ts                  | 2 +-
 client/src/document/index.tsx                 | 4 ++--
 client/src/plus/ai-help/index.tsx             | 6 +++---
 9 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/build/blog.ts b/build/blog.ts
index 5a9b4f9b6d9b..4caafa0bc912 100644
--- a/build/blog.ts
+++ b/build/blog.ts
@@ -28,7 +28,7 @@ import {
   postProcessSmallerHeadingIDs,
 } from "./utils.js";
 import { slugToFolder } from "../libs/slug-utils/index.js";
-import { codeHeaders } from "./code-headers.js";
+import { wrapCodeExamples } from "./code-headers.js";
 import { wrapTables } from "./wrap-tables.js";
 import { Doc } from "../libs/types/document.js";
 import { extractSections } from "./extract-sections.js";
@@ -391,7 +391,7 @@ export async function buildPost(
     doc.hasMathML = true;
   }
   $("div.hidden").remove();
-  codeHeaders($);
+  wrapCodeExamples($);
   injectNoTranslate($);
   injectLoadingLazyAttributes($);
   postProcessExternalLinks($);
diff --git a/build/code-headers.ts b/build/code-headers.ts
index 234e5def23dc..8b0823fb32a5 100644
--- a/build/code-headers.ts
+++ b/build/code-headers.ts
@@ -10,7 +10,7 @@ const IGNORE = new Set(["none", "text", "plain", "unix"]);
  * Mutate the `$` instance by adding headers to 
 tags containing code blocks.
  *
  */
-export function codeHeaders($: cheerio.CheerioAPI) {
+export function wrapCodeExamples($: cheerio.CheerioAPI) {
   // Our content will be like this: `
` or
   // `
` so we're technically not looking for an exact
   // match. The wildcard would technically match `
`
diff --git a/build/curriculum.ts b/build/curriculum.ts
index 7b4646534adc..068c7fc15f4e 100644
--- a/build/curriculum.ts
+++ b/build/curriculum.ts
@@ -9,7 +9,7 @@ import { DocParent } from "../libs/types/document.js";
 import { CURRICULUM_TITLE, DEFAULT_LOCALE } from "../libs/constants/index.js";
 import * as kumascript from "../kumascript/index.js";
 import LANGUAGES_RAW from "../libs/languages/index.js";
-import { codeHeaders } from "./code-headers.js";
+import { wrapCodeExamples } from "./code-headers.js";
 import {
   escapeRegExp,
   injectLoadingLazyAttributes,
@@ -321,7 +321,7 @@ export async function buildCurriculumPage(
     doc.hasMathML = true;
   }
   $("div.hidden").remove();
-  codeHeaders($);
+  wrapCodeExamples($);
   injectNoTranslate($);
   injectLoadingLazyAttributes($);
   postProcessCurriculumLinks($, (p: string | undefined) => {
diff --git a/build/index.ts b/build/index.ts
index 471d199c884e..ccacb3b2b8ef 100644
--- a/build/index.ts
+++ b/build/index.ts
@@ -28,7 +28,7 @@ import {
 } from "./flaws/index.js";
 import { checkImageReferences, checkImageWidths } from "./check-images.js";
 import { getPageTitle } from "./page-title.js";
-import { codeHeaders } from "./code-headers.js";
+import { wrapCodeExamples } from "./code-headers.js";
 import { formatNotecards } from "./format-notecards.js";
 import buildOptions from "./build-options.js";
 import LANGUAGES_RAW from "../libs/languages/index.js";
@@ -457,7 +457,7 @@ export async function buildDocument(
   }
 
   // Add headers to all 
 tags with code.
-  codeHeaders($);
+  wrapCodeExamples($);
 
   // Post process HTML so that the right elements gets tagged so they
   // *don't* get translated by tools like Google Translate.
diff --git a/client/src/blog/post.tsx b/client/src/blog/post.tsx
index b02f4e430f07..4f817184489b 100644
--- a/client/src/blog/post.tsx
+++ b/client/src/blog/post.tsx
@@ -14,7 +14,7 @@ import {
   BlogPostLimitedMetadata,
   AuthorMetadata,
 } from "../../../libs/types/blog";
-import { useDecorateExamples, useRunSample } from "../document/hooks";
+import { useDecorateCodeExamples, useRunSample } from "../document/hooks";
 import { DEFAULT_LOCALE } from "../../../libs/constants";
 import { SignUpSection as NewsletterSignUp } from "../newsletter";
 import { TOC } from "../document/organisms/toc";
@@ -187,7 +187,7 @@ export function BlogPost(props: HydrationData) {
   );
   const { doc, blogMeta } = data || props || {};
   useRunSample(doc);
-  useDecorateExamples(doc);
+  useDecorateCodeExamples(doc);
   return (
     <>
       {doc && blogMeta && (
diff --git a/client/src/document/code/syntax-highlight.tsx b/client/src/document/code/syntax-highlight.tsx
index 76acdd32cc51..520a14ad8f67 100644
--- a/client/src/document/code/syntax-highlight.tsx
+++ b/client/src/document/code/syntax-highlight.tsx
@@ -36,7 +36,7 @@ interface HighlightedCodeProps extends React.HTMLAttributes {
   children: React.ReactNode;
 }
 
-export function HighlightedElement({
+export function CodeWithSyntaxHighlight({
   language,
   children,
   ...props
diff --git a/client/src/document/hooks.ts b/client/src/document/hooks.ts
index e104aab9fa96..c75aa69515a5 100644
--- a/client/src/document/hooks.ts
+++ b/client/src/document/hooks.ts
@@ -96,7 +96,7 @@ export function useRunSample(doc: Doc | undefined) {
   }, [doc, isServer, locale]);
 }
 
-export function useDecorateExamples(doc: Doc | undefined) {
+export function useDecorateCodeExamples(doc: Doc | undefined) {
   const location = useLocation();
 
   useEffect(() => {
diff --git a/client/src/document/index.tsx b/client/src/document/index.tsx
index f67715bfef11..52ab4700bb75 100644
--- a/client/src/document/index.tsx
+++ b/client/src/document/index.tsx
@@ -6,7 +6,7 @@ import { WRITER_MODE, PLACEMENT_ENABLED } from "../env";
 import { useGA } from "../ga-context";
 import { useIsServer, useLocale } from "../hooks";
 
-import { useDocumentURL, useDecorateExamples, useRunSample } from "./hooks";
+import { useDocumentURL, useDecorateCodeExamples, useRunSample } from "./hooks";
 import { Doc } from "../../../libs/types/document";
 // Ingredients
 import { Prose } from "./ingredients/prose";
@@ -120,7 +120,7 @@ export function Document(props /* TODO: define a TS interface for this */) {
   useIncrementFrequentlyViewed(doc);
   useRunSample(doc);
   //useCollectSample(doc);
-  useDecorateExamples(doc);
+  useDecorateCodeExamples(doc);
   useInteractiveExamplesTelemetry();
 
   React.useEffect(() => {
diff --git a/client/src/plus/ai-help/index.tsx b/client/src/plus/ai-help/index.tsx
index 16fb2303eded..d0fb7d464d80 100644
--- a/client/src/plus/ai-help/index.tsx
+++ b/client/src/plus/ai-help/index.tsx
@@ -58,7 +58,7 @@ import {
 } from "./constants";
 import InternalLink from "../../ui/atoms/internal-link";
 import { isPlusSubscriber } from "../../utils";
-import { HighlightedElement } from "../../document/code/syntax-highlight";
+import { CodeWithSyntaxHighlight } from "../../document/code/syntax-highlight";
 
 type Category = "apis" | "css" | "html" | "http" | "js" | "learn";
 
@@ -484,13 +484,13 @@ function AIHelpAssistantResponse({
                 const match = /language-(\w+)/.exec(className || "");
                 const lang = match?.[1];
                 return (
-                  
                     {children}
-                  
+                  
                 );
               },
             }}

From d7ddd783ca3caccbb50473f60aee92ff5e322419 Mon Sep 17 00:00:00 2001
From: Leo McArdle 
Date: Mon, 16 Sep 2024 10:39:59 +0000
Subject: [PATCH 11/11] wip(syntax-highlighting): catch recursive errors and
 prism errors

---
 client/src/document/code/syntax-highlight.tsx | 27 ++++++++++++++-----
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/client/src/document/code/syntax-highlight.tsx b/client/src/document/code/syntax-highlight.tsx
index 520a14ad8f67..d60461d0502c 100644
--- a/client/src/document/code/syntax-highlight.tsx
+++ b/client/src/document/code/syntax-highlight.tsx
@@ -97,12 +97,21 @@ function highlightStringSync(
   const resolvedLanguage = ALIASES.get(language) || language;
   const prismLanguage = Prism.languages[resolvedLanguage];
   if (prismLanguage) {
-    return Prism.highlight(text, prismLanguage, resolvedLanguage);
+    try {
+      return Prism.highlight(text, prismLanguage, resolvedLanguage);
+    } catch {
+      console.warn("Syntax highlighting: prism error");
+    }
   }
   return;
 }
 
-async function importLanguage(language: string) {
+async function importLanguage(language: string, recursiveDepth = 0) {
+  if (recursiveDepth > 100) {
+    console.warn("Syntax highlighting: recursion error");
+    throw new Error("Syntax highlighting: recursion error");
+  }
+
   const prismLanguage = Prism.languages[language];
 
   if (!prismLanguage) {
@@ -113,7 +122,9 @@ async function importLanguage(language: string) {
           "prism-svelte"
         );
       } catch (e) {
-        console.warn(`Failed to import ${language} prism language`);
+        console.warn(
+          `Syntax highlighting: failed to import ${language} prism language`
+        );
         throw e;
       }
     } else {
@@ -124,7 +135,9 @@ async function importLanguage(language: string) {
             (typeof config.require === "string"
               ? [config.require]
               : config.require
-            ).map((dependency) => importLanguage(dependency))
+            ).map((dependency) =>
+              importLanguage(dependency, recursiveDepth + 1)
+            )
           );
         } catch {
           return;
@@ -135,7 +148,7 @@ async function importLanguage(language: string) {
           (typeof config.optional === "string"
             ? [config.optional]
             : config.optional
-          ).map((dependency) => importLanguage(dependency))
+          ).map((dependency) => importLanguage(dependency, recursiveDepth + 1))
         );
       }
       try {
@@ -145,7 +158,9 @@ async function importLanguage(language: string) {
           `prismjs/components/prism-${language}.js`
         );
       } catch (e) {
-        console.warn(`Failed to import ${language} prism language`);
+        console.warn(
+          `Syntax highlighting: failed to import ${language} prism language`
+        );
         throw e;
       }
     }