From 5b49e673a504ed56a9a06d1c72a7da0713b92b99 Mon Sep 17 00:00:00 2001
From: Mishig <dmishig@gmail.com>
Date: Thu, 30 May 2024 10:37:28 +0200
Subject: [PATCH] Escape hash in code/codeblocks (#504)

* Escape hash in code/codeblocks

* format

* pin test accelerarte
---
 .github/workflows/accelerate_doc.yml |  1 +
 kit/preprocessors/hashInCode.js      | 17 +++++++++++++++++
 kit/preprocessors/index.js           |  1 +
 kit/preprocessors/mdsvex/index.js    | 10 +---------
 kit/preprocessors/utils.js           |  7 +++++--
 kit/svelte.config.js                 |  2 ++
 6 files changed, 27 insertions(+), 11 deletions(-)
 create mode 100644 kit/preprocessors/hashInCode.js

diff --git a/.github/workflows/accelerate_doc.yml b/.github/workflows/accelerate_doc.yml
index 75a9a5b7..987cf1ec 100644
--- a/.github/workflows/accelerate_doc.yml
+++ b/.github/workflows/accelerate_doc.yml
@@ -23,6 +23,7 @@ jobs:
         with:
           repository: 'huggingface/accelerate'
           path: accelerate
+          ref: 16eb6d76bf987c7d8d877ce5152f2e29878eab37
 
       - name: Loading cache.
         uses: actions/cache@v2
diff --git a/kit/preprocessors/hashInCode.js b/kit/preprocessors/hashInCode.js
new file mode 100644
index 00000000..e7e65eed
--- /dev/null
+++ b/kit/preprocessors/hashInCode.js
@@ -0,0 +1,17 @@
+import { replaceAsync } from "./utils.js";
+
+// Preprocessor that converts `#` char inside code/codeblocks into its html5 entity `&amp;num;`
+// otherwise, an extra new space was being rendered
+export const hashInCodePreprocess = {
+	markup: async ({ content }) => {
+		const REGEX_CODE_BLOCK = /```.*?```/gs;
+		const REGEX_INLINE_CODE = /`.*?`/gs;
+		content = await replaceAsync(content, REGEX_CODE_BLOCK, async (codeContent) => {
+			return codeContent.replaceAll("#", "&amp;num;");
+		});
+		content = await replaceAsync(content, REGEX_INLINE_CODE, async (codeContent) => {
+			return codeContent.replaceAll("#", "&amp;num;");
+		});
+		return { code: content };
+	},
+};
diff --git a/kit/preprocessors/index.js b/kit/preprocessors/index.js
index 3c3b849c..721aa827 100644
--- a/kit/preprocessors/index.js
+++ b/kit/preprocessors/index.js
@@ -4,3 +4,4 @@ export { inferenceSnippetPreprocess } from "./inferenceSnippet.js";
 export { tokenizersLangPreprocess } from "./tokenizersLang.js";
 export { mdsvexPreprocess } from "./mdsvex/index.js";
 export { hfOptionsPreprocess } from "./hfOptions.js";
+export { hashInCodePreprocess } from "./hashInCode.js";
diff --git a/kit/preprocessors/mdsvex/index.js b/kit/preprocessors/mdsvex/index.js
index ecde20c2..872cd57f 100644
--- a/kit/preprocessors/mdsvex/index.js
+++ b/kit/preprocessors/mdsvex/index.js
@@ -6,6 +6,7 @@ import htmlTags from "html-tags";
 import { readdir } from "fs/promises";
 import path from "path";
 import cheerio from "cheerio";
+import { renderSvelteChars } from "../utils.js";
 
 /**
  * inside `<code>` html elements, we need to replace `&amp;` with `&`
@@ -43,15 +44,6 @@ export const mdsvexPreprocess = {
 	},
 };
 
-/**
- * Render escaped characters like `<`, `{`.
- * used for Doc
- * @param {string} code
- */
-function renderSvelteChars(code) {
-	return code.replace(/&amp;lcub;/g, "{").replace(/&amp;lt;/g, "<");
-}
-
 /**
  * Latex support in mdsvex
  * @param {string} content
diff --git a/kit/preprocessors/utils.js b/kit/preprocessors/utils.js
index 2a90e8a1..a67109c1 100644
--- a/kit/preprocessors/utils.js
+++ b/kit/preprocessors/utils.js
@@ -24,12 +24,15 @@ export async function replaceAsync(string, searchValue, replacer) {
 }
 
 /**
- * Render escaped characters like `<`, `{`.
+ * Render escaped characters like `<`, `{`, '#'.
  * used for Doc
  * @param {string} code
  */
 export function renderSvelteChars(code) {
-	return code.replace(/&amp;lcub;/g, "{").replace(/&amp;lt;/g, "<");
+	return code
+		.replace(/&amp;lcub;/g, "{")
+		.replace(/&amp;lt;/g, "<")
+		.replace(/&amp;num;/g, "#");
 }
 
 /**
diff --git a/kit/svelte.config.js b/kit/svelte.config.js
index f65470a2..90ea8e99 100644
--- a/kit/svelte.config.js
+++ b/kit/svelte.config.js
@@ -8,6 +8,7 @@ import {
 	inferenceSnippetPreprocess,
 	tokenizersLangPreprocess,
 	hfOptionsPreprocess,
+	hashInCodePreprocess,
 } from "./preprocessors/index.js";
 
 /** @type {import('@sveltejs/kit').Config} */
@@ -15,6 +16,7 @@ const config = {
 	// Consult https://kit.svelte.dev/docs/integrations#preprocessors
 	// for more information about preprocessors
 	preprocess: [
+		hashInCodePreprocess,
 		docstringPreprocess,
 		frameworkcontentPreprocess,
 		inferenceSnippetPreprocess,