From e63378e5957eb0f53771cc8f45f87c60fa16db2d Mon Sep 17 00:00:00 2001
From: Tom French <tom@tomfren.ch>
Date: Thu, 25 Jan 2024 12:19:51 +0000
Subject: [PATCH 1/8] chore: integrate code snippets script into docs

---
 docs/.gitignore                           |   2 +
 docs/docs/noir/standard_library/traits.md |  69 +----
 docs/docusaurus.config.ts                 |   1 +
 docs/package.json                         |   5 +-
 docs/scripts/preprocess/include_code.js   | 311 ++++++++++++++++++++++
 docs/scripts/preprocess/index.js          | 135 ++++++++++
 noir_stdlib/src/cmp.nr                    |   6 +-
 noir_stdlib/src/ops.nr                    |  23 +-
 8 files changed, 491 insertions(+), 61 deletions(-)
 create mode 100644 docs/scripts/preprocess/include_code.js
 create mode 100644 docs/scripts/preprocess/index.js

diff --git a/docs/.gitignore b/docs/.gitignore
index 4f6eee8284e..501e7e465ea 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -3,6 +3,8 @@
 
 # Production
 /build
+processed-docs
+processed-docs-cache
 
 # Generated files
 .docusaurus
diff --git a/docs/docs/noir/standard_library/traits.md b/docs/docs/noir/standard_library/traits.md
index f2960ca5080..841e512e7de 100644
--- a/docs/docs/noir/standard_library/traits.md
+++ b/docs/docs/noir/standard_library/traits.md
@@ -56,11 +56,8 @@ types such as arrays are filled with default values of their element type.
 
 ### `std::cmp::Eq`
 
-```rust
-trait Eq {
-    fn eq(self, other: Self) -> bool;
-}
-```
+#include_code eq-trait noir_stdlib/src/cmp.nr rust
+
 Returns `true` if `self` is equal to `other`. Implementing this trait on a type
 allows the type to be used with `==` and `!=`.
 
@@ -97,13 +94,9 @@ impl<A, B, C, D, E> Eq for (A, B, C, D, E)
     where A: Eq, B: Eq, C: Eq, D: Eq, E: Eq { .. }
 ```
 
-### `std::cmp::Cmp`
+### `std::cmp::Ord`
 
-```rust
-trait Cmp {
-    fn cmp(self, other: Self) -> Ordering;
-}
-```
+#include_code ord-trait noir_stdlib/src/cmp.nr rust
 
 `a.cmp(b)` compares two values returning `Ordering::less()` if `a < b`,
 `Ordering::equal()` if `a == b`, or `Ordering::greater()` if `a > b`.
@@ -151,23 +144,10 @@ These traits abstract over addition, subtraction, multiplication, and division r
 Implementing these traits for a given type will also allow that type to be used with the corresponding operator
 for that trait (`+` for Add, etc) in addition to the normal method names.
 
-```rust
-trait Add {
-    fn add(self, other: Self) -> Self;
-}
-
-trait Sub {
-    fn sub(self, other: Self) -> Self;
-}
-
-trait Mul {
-    fn mul(self, other: Self) -> Self;
-}
-
-trait Div {
-    fn div(self, other: Self) -> Self;
-}
-```
+#include_code add-trait noir_stdlib/src/ops.nr rust
+#include_code sub-trait noir_stdlib/src/ops.nr rust
+#include_code mul-trait noir_stdlib/src/ops.nr rust
+#include_code div-trait noir_stdlib/src/ops.nr rust
 
 The implementations block below is given for the `Add` trait, but the same types that implement
 `Add` also implement `Sub`, `Mul`, and `Div`.
@@ -189,11 +169,7 @@ impl Add for u64 { .. }
 
 ### `std::ops::Rem`
 
-```rust
-trait Rem {
-    fn rem(self, other: Self) -> Self;
-}
-```
+#include_code rem-trait noir_stdlib/src/ops.nr rust
 
 `Rem::rem(a, b)` is the remainder function returning the result of what is
 left after dividing `a` and `b`. Implementing `Rem` allows the `%` operator
@@ -216,19 +192,9 @@ impl Rem for i64 { fn rem(self, other: i64) -> i64 { self % other } }
 
 ### `std::ops::{ BitOr, BitAnd, BitXor }`
 
-```rust
-trait BitOr {
-    fn bitor(self, other: Self) -> Self;
-}
-
-trait BitAnd {
-    fn bitand(self, other: Self) -> Self;
-}
-
-trait BitXor {
-    fn bitxor(self, other: Self) -> Self;
-}
-```
+#include_code bitor-trait noir_stdlib/src/ops.nr rust
+#include_code bitand-trait noir_stdlib/src/ops.nr rust
+#include_code bitxor-trait noir_stdlib/src/ops.nr rust
 
 Traits for the bitwise operations `|`, `&`, and `^`.
 
@@ -255,15 +221,8 @@ impl BitOr for i64 { fn bitor(self, other: i64) -> i64 { self | other } }
 
 ### `std::ops::{ Shl, Shr }`
 
-```rust
-trait Shl {
-    fn shl(self, other: Self) -> Self;
-}
-
-trait Shr {
-    fn shr(self, other: Self) -> Self;
-}
-```
+#include_code shl-trait noir_stdlib/src/ops.nr rust
+#include_code shr-trait noir_stdlib/src/ops.nr rust
 
 Traits for a bit shift left and bit shift right.
 
diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts
index aacc318f5be..4e0d053f61e 100644
--- a/docs/docusaurus.config.ts
+++ b/docs/docusaurus.config.ts
@@ -26,6 +26,7 @@ export default {
       '@docusaurus/preset-classic',
       {
         docs: {
+          path: "processed-docs",
           sidebarPath: './sidebars.js',
           routeBasePath: '/docs',
           remarkPlugins: [math],
diff --git a/docs/package.json b/docs/package.json
index 1e3efcfe3d1..6c706e4f514 100644
--- a/docs/package.json
+++ b/docs/package.json
@@ -3,8 +3,9 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
-    "start": "docusaurus start",
-    "build": "yarn version::stables && docusaurus build",
+    "preprocess": "yarn node ./scripts/preprocess/index.js",
+    "start": "yarn preprocess && docusaurus start",
+    "build": "yarn preprocess && yarn version::stables && docusaurus build",
     "version::stables": "ts-node ./scripts/setStable.ts",
     "serve": "serve build"
   },
diff --git a/docs/scripts/preprocess/include_code.js b/docs/scripts/preprocess/include_code.js
new file mode 100644
index 00000000000..3359cb167c8
--- /dev/null
+++ b/docs/scripts/preprocess/include_code.js
@@ -0,0 +1,311 @@
+const fs = require('fs');
+const path = require('path');
+const childProcess = require('child_process');
+
+const getLineNumberFromIndex = (fileContent, index) => {
+  return fileContent.substring(0, index).split('\n').length;
+};
+
+/**
+ * Search for lines of the form
+ */
+function processHighlighting(codeSnippet, identifier) {
+  const lines = codeSnippet.split('\n');
+  /**
+   * For an identifier = bar:
+   *
+   * Matches of the form: `highlight-next-line:foo:bar:baz` will be replaced with "highlight-next-line".
+   * Matches of the form: `highlight-next-line:foo:baz` will be replaced with "".
+   */
+  const regex1 = /highlight-next-line:([a-zA-Z0-9-._:]+)/;
+  const replacement1 = 'highlight-next-line';
+  const regex2 = /highlight-start:([a-zA-Z0-9-._:]+)/;
+  const replacement2 = 'highlight-start';
+  const regex3 = /highlight-end:([a-zA-Z0-9-._:]+)/;
+  const replacement3 = 'highlight-end';
+  const regex4 = /this-will-error:([a-zA-Z0-9-._:]+)/;
+  const replacement4 = 'this-will-error';
+
+  let result = '';
+  let mutated = false;
+
+  const processLine = (line, regex, replacement) => {
+    const match = line.match(regex);
+    if (match) {
+      mutated = true;
+
+      const identifiers = match[1].split(':');
+      if (identifiers.includes(identifier)) {
+        line = line.replace(match[0], replacement);
+      } else {
+        // Remove matched text completely
+        line = line.replace(match[0], '');
+      }
+    } else {
+      // No match: it's an ordinary line of code.
+    }
+    return line.trim() == '//' || line.trim() == '#' ? '' : line;
+  };
+
+  for (let line of lines) {
+    mutated = false;
+    line = processLine(line, regex1, replacement1);
+    line = processLine(line, regex2, replacement2);
+    line = processLine(line, regex3, replacement3);
+    line = processLine(line, regex4, replacement4);
+    result += line === '' && mutated ? '' : line + '\n';
+  }
+
+  return result.trim();
+}
+
+let lastReleasedVersion;
+
+/** Returns the last released tag */
+function getLatestTag() {
+  if (!lastReleasedVersion) {
+    const manifest = path.resolve(__dirname, '../../../.release-please-manifest.json');
+    lastReleasedVersion = JSON.parse(fs.readFileSync(manifest).toString())['.'];
+  }
+  return lastReleasedVersion ? `v${lastReleasedVersion}` : undefined;
+}
+
+/** Returns whether to use the latest release or the current version of stuff. */
+function useLastRelease() {
+  return process.env.NETLIFY || process.env.INCLUDE_RELEASED_CODE;
+}
+
+/**
+ * Returns the contents of a file. If the build is running for publishing, it will load the contents
+ * of the file in the last released version.
+ */
+function readFile(filePath, tag) {
+  if (tag && tag !== 'master') {
+    try {
+      const root = path.resolve(__dirname, '../../../');
+      const relPath = path.relative(root, filePath);
+      return childProcess.execSync(`git show ${tag}:${relPath}`).toString();
+    } catch (err) {
+      console.error(`Error reading file ${filePath} from version ${tag}. Falling back to current content.`);
+    }
+  }
+  return fs.readFileSync(filePath, 'utf-8');
+}
+
+/** Extracts a code snippet, trying with the last release if applicable, and falling back to current content. */
+function extractCodeSnippet(filePath, identifier, requesterFile) {
+  if (useLastRelease()) {
+    try {
+      return doExtractCodeSnippet(filePath, identifier, false);
+    } catch (err) {
+      console.error(
+        `Error extracting code snippet ${identifier} from ${path.basename(
+          filePath,
+        )} requested by ${requesterFile}: ${err}. Falling back to current content.`,
+      );
+    }
+  }
+
+  return doExtractCodeSnippet(filePath, identifier, true);
+}
+
+/**
+ * Parse a code file, looking for identifiers of the form:
+ * `docs:start:${identifier}` and `docs:end:{identifier}`.
+ * Extract that section of code.
+ *
+ * It's complicated if code snippet identifiers overlap (i.e. the 'start' of one code snippet is in the
+ * middle of another code snippet). The extra logic in this function searches for all identifiers, and
+ * removes any which fall within the bounds of the code snippet for this particular `identifier` param.
+ * @returns the code snippet, and start and end line numbers which can later be used for creating a link to github source code.
+ */
+function doExtractCodeSnippet(filePath, identifier, useCurrent) {
+  const tag = useCurrent ? 'master' : getLatestTag();
+  let fileContent = readFile(filePath, tag);
+  let lineRemovalCount = 0;
+  let linesToRemove = [];
+
+  const startRegex = /(?:\/\/|#)\s+docs:start:([a-zA-Z0-9-._:]+)/g; // `g` will iterate through the regex.exec loop
+  const endRegex = /(?:\/\/|#)\s+docs:end:([a-zA-Z0-9-._:]+)/g;
+
+  /**
+   * Search for one of the regex statements in the code file. If it's found, return the line as a string and the line number.
+   */
+  const lookForMatch = (regex) => {
+    let match;
+    let matchFound = false;
+    let matchedLineNum = null;
+    let actualMatch = null;
+    let lines = fileContent.split('\n');
+    while ((match = regex.exec(fileContent))) {
+      if (match !== null) {
+        const identifiers = match[1].split(':');
+        let tempMatch = identifiers.includes(identifier) ? match : null;
+
+        if (tempMatch === null) {
+          // If it's not a match, we'll make a note that we should remove the matched text, because it's from some other identifier and should not appear in the snippet for this identifier.
+          for (let i = 0; i < lines.length; i++) {
+            let line = lines[i];
+            if (line.trim() == match[0].trim()) {
+              linesToRemove.push(i + 1); // lines are indexed from 1
+              ++lineRemovalCount;
+            }
+          }
+        } else {
+          if (matchFound === true) {
+            throw new Error(`Duplicate for regex ${regex} and identifier ${identifier}`);
+          }
+          matchFound = true;
+          matchedLineNum = getLineNumberFromIndex(fileContent, tempMatch.index);
+          actualMatch = tempMatch;
+        }
+      }
+    }
+
+    return [actualMatch, matchedLineNum];
+  };
+
+  let [startMatch, startLineNum] = lookForMatch(startRegex);
+  let [endMatch, endLineNum] = lookForMatch(endRegex);
+
+  // Double-check that the extracted line actually contains the required start and end identifier.
+  if (startMatch !== null) {
+    const startIdentifiers = startMatch[1].split(':');
+    startMatch = startIdentifiers.includes(identifier) ? startMatch : null;
+  }
+  if (endMatch !== null) {
+    const endIdentifiers = endMatch[1].split(':');
+    endMatch = endIdentifiers.includes(identifier) ? endMatch : null;
+  }
+
+  if (startMatch === null || endMatch === null) {
+    if (startMatch === null && endMatch === null) {
+      throw new Error(`Identifier "${identifier}" not found in file "${filePath}"`);
+    } else if (startMatch === null) {
+      throw new Error(`Start line "docs:start:${identifier}" not found in file "${filePath}"`);
+    } else {
+      throw new Error(`End line "docs:end:${identifier}" not found in file "${filePath}"`);
+    }
+  }
+
+  let lines = fileContent.split('\n');
+
+  // We only want to remove lines which actually fall within the bounds of our code snippet, so narrow down the list of lines that we actually want to remove.
+  linesToRemove = linesToRemove.filter((lineNum) => {
+    const removal_in_bounds = lineNum >= startLineNum && lineNum <= endLineNum;
+    return removal_in_bounds;
+  });
+
+  // Remove lines which contain `docs:` comments for unrelated identifiers:
+  lines = lines.filter((l, i) => {
+    return !linesToRemove.includes(i + 1); // lines are indexed from 1
+  });
+
+  // Remove lines from the snippet which fall outside the `docs:start` and `docs:end` values.
+  lines = lines.filter((l, i) => {
+    return i + 1 > startLineNum && i + 1 < endLineNum - linesToRemove.length; // lines are indexed from 1
+  });
+
+  // We have our code snippet!
+  let codeSnippet = lines.join('\n');
+
+  // The code snippet might contain some docusaurus highlighting comments for other identifiers. We should remove those.
+  codeSnippet = processHighlighting(codeSnippet, identifier);
+
+  return [codeSnippet, startLineNum, endLineNum, tag];
+}
+
+/**
+ * Explaining this regex:
+ *
+ * E.g. `#include_code snippet_identifier /circuits/my_code.cpp cpp`
+ *
+ * #include_code\s+(\S+)\s+(\S+)\s+(\S+)
+ *   - This is the main regex to match the above format.
+ *   - \s+: one or more whitespace characters (space or tab) after `include_code` command.
+ *   - (\S+): one or more non-whitespaced characters. Captures this as the first argument, which is a human-readable identifier for the code block.
+ *   - etc.
+ *
+ * Lookaheads are needed to allow us to ignore commented-out lines:
+ *
+ * ^(?!<!--.*)
+ *   - ^: Asserts the beginning of the line.
+ *   - (?!<!--.*): Negative lookahead assertion to ensure the line does not start with markdown comment syntax `<!--`.
+ *
+ * (?=.*STUFF)
+ *   - Positive lookahead assertion to ensure the line contains the command (STUFF) we want to match.
+ *
+ * .*$
+ *   - .*: Matches any characters (except newline) in the line.
+ *   - $: Asserts the end of the line.
+ *
+ * `/gm`
+ *   - match globally (g) across the entire input text and consider multiple lines (m) when matching. This is necessary to handle multiple include tags throughout the markdown content.
+ */
+const regex = /^(?!<!--.*)(?=.*#include_code\s+(\S+)\s+(\S+)\s+(\S+)(?:[ ]+(\S+))?).*$/gm;
+
+async function preprocessIncludeCode(markdownContent, filePath, rootDir) {
+  // Process each include tag in the current markdown file
+  let updatedContent = markdownContent;
+  let matchesFound = false;
+  let match;
+  while ((match = regex.exec(markdownContent))) {
+    matchesFound = true;
+    const fullMatch = match[0];
+    const identifier = match[1];
+    let codeFilePath = match[2];
+    const language = match[3];
+    const opts = match[4] || '';
+
+    if (codeFilePath.slice(0) != '/') {
+      // Absolute path to the code file from the root of the Docusaurus project
+      // Note: without prefixing with `/`, the later call to `path.resolve()` gives an incorrect path (absolute instead of relative)
+      codeFilePath = `/${codeFilePath}`;
+    }
+
+    const noTitle = opts.includes('noTitle');
+    const noLineNumbers = opts.includes('noLineNumbers');
+    const noSourceLink = opts.includes('noSourceLink');
+
+    try {
+      const absCodeFilePath = path.join(rootDir, codeFilePath);
+
+      // Extract the code snippet between the specified comments
+      const extracted = extractCodeSnippet(absCodeFilePath, identifier, filePath);
+      const [codeSnippet, startLine, endLine, tag] = extracted;
+
+      const relativeCodeFilePath = path.resolve(rootDir, codeFilePath);
+
+      let urlText = `${relativeCodeFilePath}#L${startLine}-L${endLine}`;
+      if (tag && tag !== 'master') urlText += ` (${tag})`;
+      const url = `https://github.com/noir-lang/noir/blob/${tag}/${relativeCodeFilePath}#L${startLine}-L${endLine}`;
+
+      const title = noTitle ? '' : `title="${identifier}"`;
+      const lineNumbers = noLineNumbers ? '' : 'showLineNumbers';
+      const warn =
+        useLastRelease() && (!tag || tag === 'master')
+          ? `<br/>This example references unreleased code. Code from released packages may be different. Use with care.`
+          : '';
+      const source = noSourceLink
+        ? ''
+        : `\n> <sup><sub><a href="${url}" target="_blank" rel="noopener noreferrer">Source code: ${urlText}</a>${warn}</sub></sup>`;
+      const replacement =
+        language === 'raw'
+          ? codeSnippet
+          : `\`\`\`${language} ${title} ${lineNumbers} \n${codeSnippet}\n\`\`\`${source}\n`;
+
+      // Replace the include tag with the code snippet
+      updatedContent = updatedContent.replace(fullMatch, replacement);
+    } catch (error) {
+      const lineNum = getLineNumberFromIndex(markdownContent, match.index);
+      // We were warning here, but code snippets were being broken. So making this throw an error instead:
+      throw new Error(`Error processing "${filePath}:${lineNum}": ${error.message}.`);
+    }
+  }
+
+  return { content: updatedContent, isUpdated: matchesFound };
+}
+
+module.exports = {
+  preprocessIncludeCode,
+};
diff --git a/docs/scripts/preprocess/index.js b/docs/scripts/preprocess/index.js
new file mode 100644
index 00000000000..c507fc6412b
--- /dev/null
+++ b/docs/scripts/preprocess/index.js
@@ -0,0 +1,135 @@
+const fs = require('fs');
+const path = require('path');
+
+const { preprocessIncludeCode } = require('./include_code');
+
+async function processMarkdownFilesInDir(rootDir, docsDir, regex) {
+  const files = fs.readdirSync(docsDir);
+  const contentUpdates = [];
+
+  for (const file of files) {
+    const filepath = path.join(docsDir, file);
+    const stat = fs.statSync(filepath);
+
+    if (stat.isDirectory()) {
+      contentUpdates.push(processMarkdownFilesInDir(rootDir, filepath, regex));
+    } else if (stat.isFile() && (file.endsWith('.md') || file.endsWith('.mdx'))) {
+      const markdownContent = fs.readFileSync(filepath, 'utf-8');
+
+      let updatedContent = markdownContent;
+      let isUpdated = false;
+      for (preprocess of [preprocessIncludeCode]) {
+        const result = await preprocess(updatedContent, filepath, rootDir);
+        updatedContent = result.content;
+        isUpdated = isUpdated || result.isUpdated;
+      }
+
+      contentUpdates.push({
+        content: updatedContent,
+        filepath,
+        isUpdated,
+      });
+    }
+  }
+
+  return Promise.all(contentUpdates);
+}
+
+async function writeProcessedFiles(docsDir, destDir, cachedDestDir, content) {
+  let writePromises = [];
+
+  if (Array.isArray(content)) {
+    // It's a dir
+    if (content.length > 0) {
+      // It's a nonempty dir
+      writePromises.push(
+        await Promise.all(content.map((a) => writeProcessedFiles(docsDir, destDir, cachedDestDir, a))),
+      );
+    } else {
+      // empty dir
+    }
+  } else if (!content.filepath) {
+    // Do nothing
+  } else {
+    // It's a file
+    // Derive the destination path from the original path:
+    const relPath = path.relative(docsDir, content.filepath);
+    const destFilePath = path.resolve(destDir, relPath);
+    const destDirName = path.dirname(destFilePath);
+    const cachedDestFilePath = path.resolve(cachedDestDir, relPath);
+    const cachedDestDirName = path.dirname(cachedDestFilePath);
+
+    if (!fs.existsSync(destDirName)) {
+      fs.mkdirSync(destDirName, { recursive: true });
+    }
+    if (!fs.existsSync(cachedDestDirName)) {
+      fs.mkdirSync(cachedDestDirName, { recursive: true });
+    }
+
+    // If the file exists, don't overwrite unless we need to:
+    if (fs.existsSync(destFilePath)) {
+      const existingFileContent = fs.readFileSync(destFilePath, 'utf-8');
+
+      // Safety: try and check whether the dev has been making code edits in the wrong dir!
+      if (fs.existsSync(cachedDestFilePath)) {
+        const cachedFileContent = fs.readFileSync(cachedDestFilePath, 'utf-8');
+        if (existingFileContent !== cachedFileContent) {
+          throw new Error(
+            `It looks like you might have accidentally edited files in the 'processed-docs/' dir instead of the 'docs/' dir (because there's a discrepancy between 'preprocessed-docs' and 'preprocessed-docs-cache', but they should always be the same unless they're tampered-with).\n\nWe don't want you to accidentally overwrite your work.\n\nCopy your work to the 'docs/' dir, and revert your 'processed-docs/' changes.\n\nI.e. copy from here: ${destFilePath}\n\nto here: ${content.filepath}\n\nIf this error's safety assumption is wrong, and you'd like to proceed with building, please delete the cached file ${cachedDestFilePath} and rerun the build.\n\nAnd if you've not made any changes at all to the docs and you've just pulled master and are wondering what is going on, you might want to run \`yarn clear\` from this docs dir.`,
+          );
+        }
+      }
+
+      // Don't write if no change.
+      if (existingFileContent === content.content) {
+        // Do nothing: the content doesn't need to be overwritten.
+        // This will speed up the docusaurus build.
+        return;
+      }
+    }
+
+    writePromises.push(
+      fs.promises.writeFile(destFilePath, content.content, {
+        encoding: 'utf8',
+        flag: 'w', // overwrite
+      }),
+    );
+
+    // Cache the dest data as well, as a safety measure, to ensure no one edits the processed-docs instead of the docs, by mistake.
+    writePromises.push(
+      fs.promises.writeFile(cachedDestFilePath, content.content, {
+        encoding: 'utf8',
+        flag: 'w', // overwrite
+      }),
+    );
+  }
+
+  return Promise.all(writePromises);
+}
+
+async function run() {
+  const rootDir = path.join(__dirname, '../../../');
+  const docsDir = path.join(rootDir, 'docs', 'docs');
+  const destDir = path.join(rootDir, 'docs', 'processed-docs');
+  const cachedDestDir = path.join(rootDir, 'docs', 'processed-docs-cache');
+
+  const content = await processMarkdownFilesInDir(rootDir, docsDir);
+
+  await writeProcessedFiles(docsDir, destDir, cachedDestDir, content);
+
+  console.log('Preprocessing complete.');
+}
+
+/**
+ * Parses all .md and .mdx files in docs/ for lines of the form:
+ *   #include_code snippet_identifier /circuits/my_code.cpp cpp
+ *
+ * Reads the code file and extracts the code snippet bookended by `docs:start:snippet_identifier` and `docs:end:snippet_identifier.
+ *
+ * Replaces the `#include_code` line with the code snippet (in memory).
+ *
+ * Writes the updated .md or .mdx file to a `processed-docs/` dir.
+ *
+ * docusaurus then can build from the `processed-docs/` dir.
+ */
+run();
diff --git a/noir_stdlib/src/cmp.nr b/noir_stdlib/src/cmp.nr
index 11127494c18..b3de3e2658e 100644
--- a/noir_stdlib/src/cmp.nr
+++ b/noir_stdlib/src/cmp.nr
@@ -1,6 +1,8 @@
+// docs:start:eq-trait
 trait Eq {
     fn eq(self, other: Self) -> bool;
 }
+// docs:end:eq-trait
 
 impl Eq for Field { fn eq(self, other: Field) -> bool { self == other } }
 
@@ -66,7 +68,6 @@ impl Eq for Ordering {
     }
 }
 
-
 // Noir doesn't have enums yet so we emulate (Lt | Eq | Gt) with a struct
 // that has 3 public functions for constructing the struct.
 struct Ordering {
@@ -90,10 +91,11 @@ impl Ordering {
     }
 }
 
-
+// docs:start:ord-trait
 trait Ord {
     fn cmp(self, other: Self) -> Ordering;
 }
+// docs:end:ord-trait
 
 // Note: Field deliberately does not implement Ord
 
diff --git a/noir_stdlib/src/ops.nr b/noir_stdlib/src/ops.nr
index 3078ac11296..50386290b8e 100644
--- a/noir_stdlib/src/ops.nr
+++ b/noir_stdlib/src/ops.nr
@@ -1,7 +1,8 @@
-
+// docs:start:add-trait
 trait Add {
     fn add(self, other: Self) -> Self;
 }
+// docs:end:add-trait
 
 impl Add for Field { fn add(self, other: Field) -> Field { self + other } }
 
@@ -15,9 +16,11 @@ impl Add for i16 { fn add(self, other: i16) -> i16 { self + other } }
 impl Add for i32 { fn add(self, other: i32) -> i32 { self + other } }
 impl Add for i64 { fn add(self, other: i64) -> i64 { self + other } }
 
+// docs:start:sub-trait
 trait Sub {
     fn sub(self, other: Self) -> Self;
 }
+// docs:end:sub-trait
 
 impl Sub for Field { fn sub(self, other: Field) -> Field { self - other } }
 
@@ -31,9 +34,11 @@ impl Sub for i16 { fn sub(self, other: i16) -> i16 { self - other } }
 impl Sub for i32 { fn sub(self, other: i32) -> i32 { self - other } }
 impl Sub for i64 { fn sub(self, other: i64) -> i64 { self - other } }
 
+// docs:start:mul-trait
 trait Mul {
     fn mul(self, other: Self) -> Self;
 }
+// docs:end:mul-trait
 
 impl Mul for Field { fn mul(self, other: Field) -> Field { self * other } }
 
@@ -47,9 +52,11 @@ impl Mul for i16 { fn mul(self, other: i16) -> i16 { self * other } }
 impl Mul for i32 { fn mul(self, other: i32) -> i32 { self * other } }
 impl Mul for i64 { fn mul(self, other: i64) -> i64 { self * other } }
 
+// docs:start:div-trait
 trait Div {
     fn div(self, other: Self) -> Self;
 }
+// docs:end:div-trait
 
 impl Div for Field { fn div(self, other: Field) -> Field { self / other } }
 
@@ -63,9 +70,11 @@ impl Div for i16 { fn div(self, other: i16) -> i16 { self / other } }
 impl Div for i32 { fn div(self, other: i32) -> i32 { self / other } }
 impl Div for i64 { fn div(self, other: i64) -> i64 { self / other } }
 
-trait Rem {
+// docs:start:rem-trait
+trait Rem{
     fn rem(self, other: Self) -> Self;
 }
+// docs:end:rem-trait
 
 impl Rem for u8 { fn rem(self, other: u8) -> u8 { self % other } }
 impl Rem for u16 { fn rem(self, other: u16) -> u16 { self % other } }
@@ -77,9 +86,11 @@ impl Rem for i16 { fn rem(self, other: i16) -> i16 { self % other } }
 impl Rem for i32 { fn rem(self, other: i32) -> i32 { self % other } }
 impl Rem for i64 { fn rem(self, other: i64) -> i64 { self % other } }
 
+// docs:start:bitor-trait
 trait BitOr {
     fn bitor(self, other: Self) -> Self;
 }
+// docs:end:bitor-trait
 
 impl BitOr for bool { fn bitor(self, other: bool) -> bool { self | other } }
 
@@ -93,9 +104,11 @@ impl BitOr for i16 { fn bitor(self, other: i16) -> i16 { self | other } }
 impl BitOr for i32 { fn bitor(self, other: i32) -> i32 { self | other } }
 impl BitOr for i64 { fn bitor(self, other: i64) -> i64 { self | other } }
 
+// docs:start:bitand-trait
 trait BitAnd {
     fn bitand(self, other: Self) -> Self;
 }
+// docs:end:bitand-trait
 
 impl BitAnd for bool { fn bitand(self, other: bool) -> bool { self & other } }
 
@@ -109,9 +122,11 @@ impl BitAnd for i16 { fn bitand(self, other: i16) -> i16 { self & other } }
 impl BitAnd for i32 { fn bitand(self, other: i32) -> i32 { self & other } }
 impl BitAnd for i64 { fn bitand(self, other: i64) -> i64 { self & other } }
 
+// docs:start:bitxor-trait
 trait BitXor {
     fn bitxor(self, other: Self) -> Self;
 }
+// docs:end:bitxor-trait
 
 impl BitXor for bool { fn bitxor(self, other: bool) -> bool { self ^ other } }
 
@@ -125,9 +140,11 @@ impl BitXor for i16 { fn bitxor(self, other: i16) -> i16 { self ^ other } }
 impl BitXor for i32 { fn bitxor(self, other: i32) -> i32 { self ^ other } }
 impl BitXor for i64 { fn bitxor(self, other: i64) -> i64 { self ^ other } }
 
+// docs:start:shl-trait
 trait Shl {
     fn shl(self, other: Self) -> Self;
 }
+// docs:end:shl-trait
 
 impl Shl for u8 { fn shl(self, other: u8) -> u8 { self << other } }
 impl Shl for u16 { fn shl(self, other: u16) -> u16 { self << other } }
@@ -140,9 +157,11 @@ impl Shl for u64 { fn shl(self, other: u64) -> u64 { self << other } }
 // impl Shl for i32 { fn shl(self, other: i32) -> i32 { self << other } }
 // impl Shl for i64 { fn shl(self, other: i64) -> i64 { self << other } }
 
+// docs:start:shr-trait
 trait Shr {
     fn shr(self, other: Self) -> Self;
 }
+// docs:end:shr-trait
 
 impl Shr for u8 { fn shr(self, other: u8) -> u8 { self >> other } }
 impl Shr for u16 { fn shr(self, other: u16) -> u16 { self >> other } }

From 00b1318ecbb0fa36358ad4e9f1abd8da978d7549 Mon Sep 17 00:00:00 2001
From: Tom French <tom@tomfren.ch>
Date: Thu, 25 Jan 2024 13:06:13 +0000
Subject: [PATCH 2/8] chore: copy across json files as well

---
 docs/scripts/preprocess/index.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/scripts/preprocess/index.js b/docs/scripts/preprocess/index.js
index c507fc6412b..442b6038af1 100644
--- a/docs/scripts/preprocess/index.js
+++ b/docs/scripts/preprocess/index.js
@@ -29,6 +29,12 @@ async function processMarkdownFilesInDir(rootDir, docsDir, regex) {
         filepath,
         isUpdated,
       });
+    } else {
+      contentUpdates.push({
+        content: fs.readFileSync(filepath, 'utf-8'),
+        filepath,
+        isUpdated: false,
+      });
     }
   }
 

From baa3d9dfff9289d47637c9bf72447601363b9775 Mon Sep 17 00:00:00 2001
From: Tom French <tom@tomfren.ch>
Date: Thu, 25 Jan 2024 13:49:16 +0000
Subject: [PATCH 3/8] chore: satisfy github security

---
 docs/scripts/preprocess/include_code.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/scripts/preprocess/include_code.js b/docs/scripts/preprocess/include_code.js
index 3359cb167c8..ffe50065002 100644
--- a/docs/scripts/preprocess/include_code.js
+++ b/docs/scripts/preprocess/include_code.js
@@ -84,7 +84,8 @@ function readFile(filePath, tag) {
     try {
       const root = path.resolve(__dirname, '../../../');
       const relPath = path.relative(root, filePath);
-      return childProcess.execSync(`git show ${tag}:${relPath}`).toString();
+      const taggedPath = `${tag}:${relPath}`;
+      return childProcess.execSync('git show', taggedPath).toString();
     } catch (err) {
       console.error(`Error reading file ${filePath} from version ${tag}. Falling back to current content.`);
     }
@@ -274,7 +275,7 @@ async function preprocessIncludeCode(markdownContent, filePath, rootDir) {
       const extracted = extractCodeSnippet(absCodeFilePath, identifier, filePath);
       const [codeSnippet, startLine, endLine, tag] = extracted;
 
-      const relativeCodeFilePath = path.resolve(rootDir, codeFilePath);
+      const relativeCodeFilePath = path.resolve(rootDir, codeFilePath).slice(1);
 
       let urlText = `${relativeCodeFilePath}#L${startLine}-L${endLine}`;
       if (tag && tag !== 'master') urlText += ` (${tag})`;

From 5f2fe598b6aafd3ee62099252c0788ab286e7c0b Mon Sep 17 00:00:00 2001
From: Tom French <tom@tomfren.ch>
Date: Thu, 25 Jan 2024 14:09:42 +0000
Subject: [PATCH 4/8] chore: use code snippets with test program

---
 .../standard_library/cryptographic_primitives/hashes.mdx | 7 +------
 test_programs/execution_success/pedersen_hash/Nargo.toml | 6 ++++++
 .../execution_success/pedersen_hash/Prover.toml          | 4 ++++
 .../execution_success/pedersen_hash/src/main.nr          | 9 +++++++++
 4 files changed, 20 insertions(+), 6 deletions(-)
 create mode 100644 test_programs/execution_success/pedersen_hash/Nargo.toml
 create mode 100644 test_programs/execution_success/pedersen_hash/Prover.toml
 create mode 100644 test_programs/execution_success/pedersen_hash/src/main.nr

diff --git a/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx b/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx
index 3c5f7f79603..dff837473b9 100644
--- a/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx
+++ b/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx
@@ -58,12 +58,7 @@ fn pedersen_hash(_input : [Field]) -> Field
 
 example:
 
-```rust
-fn main() {
-    let x = [163, 117, 178, 149]; // some random bytes
-    let hash = std::hash::pedersen_hash(x);
-}
-```
+#include_code pedersen-hash test_programs/execution_success/pedersen_hash/src/main.nr rust
 
 <BlackBoxInfo />
 
diff --git a/test_programs/execution_success/pedersen_hash/Nargo.toml b/test_programs/execution_success/pedersen_hash/Nargo.toml
new file mode 100644
index 00000000000..6248a96b3c9
--- /dev/null
+++ b/test_programs/execution_success/pedersen_hash/Nargo.toml
@@ -0,0 +1,6 @@
+[package]
+name = "pedersen_hash"
+type = "bin"
+authors = [""]
+
+[dependencies]
diff --git a/test_programs/execution_success/pedersen_hash/Prover.toml b/test_programs/execution_success/pedersen_hash/Prover.toml
new file mode 100644
index 00000000000..931b121fa6a
--- /dev/null
+++ b/test_programs/execution_success/pedersen_hash/Prover.toml
@@ -0,0 +1,4 @@
+x = "0"
+y = "1"
+
+expected_hash = "0x0d98561fb02ca04d00801dfdc118b2a24cea0351963587712a28d368041370e1"
diff --git a/test_programs/execution_success/pedersen_hash/src/main.nr b/test_programs/execution_success/pedersen_hash/src/main.nr
new file mode 100644
index 00000000000..20c7de12d6c
--- /dev/null
+++ b/test_programs/execution_success/pedersen_hash/src/main.nr
@@ -0,0 +1,9 @@
+// docs:start:pedersen-hash
+use dep::std;
+
+fn main(x: Field, y: Field, expected_hash: Field) {
+    let hash = std::hash::pedersen_hash([x, y]);
+    assert_eq(hash, expected_hash);
+}
+// docs:end:pedersen-hash
+

From 7229044049b1b35fc4196367cfca1976c47874b3 Mon Sep 17 00:00:00 2001
From: Tom French <tom@tomfren.ch>
Date: Thu, 25 Jan 2024 14:14:41 +0000
Subject: [PATCH 5/8] chore: add rust cache

---
 .github/workflows/docs-pr.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml
index f4a1be826a8..87bec37c438 100644
--- a/.github/workflows/docs-pr.yml
+++ b/.github/workflows/docs-pr.yml
@@ -54,6 +54,15 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
     
+      - name: Setup toolchain
+        uses: dtolnay/rust-toolchain@1.71.1
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: x86_64-unknown-linux-gnu
+          cache-on-failure: false
+          save-if: false
+
       - name: Install Yarn dependencies
         uses: ./.github/actions/setup
 

From 5d07e40f48085015b868905f0da31b9af7a01449 Mon Sep 17 00:00:00 2001
From: Tom French <tom@tomfren.ch>
Date: Thu, 25 Jan 2024 14:31:20 +0000
Subject: [PATCH 6/8] chore: remove double `<BlackBoxInfo />`

---
 .../noir/standard_library/cryptographic_primitives/hashes.mdx    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx b/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx
index dff837473b9..2cd08b99f9a 100644
--- a/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx
+++ b/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx
@@ -62,7 +62,6 @@ example:
 
 <BlackBoxInfo />
 
-<BlackBoxInfo />
 
 ## pedersen_commitment
 

From 7549dba7a7921a66836303406f6856612be96b3a Mon Sep 17 00:00:00 2001
From: Tom French <tom@tomfren.ch>
Date: Thu, 25 Jan 2024 14:38:39 +0000
Subject: [PATCH 7/8] chore: add more tests into docs

---
 .../cryptographic_primitives/hashes.mdx       | 23 +++----------------
 .../execution_success/keccak256/src/main.nr   |  7 +++---
 .../pedersen_commitment/Nargo.toml            |  6 +++++
 .../pedersen_commitment/Prover.toml           |  4 ++++
 .../pedersen_commitment/src/main.nr           | 10 ++++++++
 .../poseidon_bn254_hash/src/main.nr           |  2 ++
 6 files changed, 29 insertions(+), 23 deletions(-)
 create mode 100644 test_programs/execution_success/pedersen_commitment/Nargo.toml
 create mode 100644 test_programs/execution_success/pedersen_commitment/Prover.toml
 create mode 100644 test_programs/execution_success/pedersen_commitment/src/main.nr

diff --git a/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx b/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx
index 2cd08b99f9a..3d33630bc1a 100644
--- a/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx
+++ b/docs/docs/noir/standard_library/cryptographic_primitives/hashes.mdx
@@ -73,12 +73,7 @@ fn pedersen_commitment(_input : [Field]) -> [Field; 2]
 
 example:
 
-```rust
-fn main() {
-    let x = [163, 117, 178, 149]; // some random bytes
-    let commitment = std::hash::pedersen_commitment(x);
-}
-```
+#include_code pedersen-commitment test_programs/execution_success/pedersen_commitment/src/main.nr rust
 
 <BlackBoxInfo />
 
@@ -94,13 +89,7 @@ fn keccak256<N>(_input : [u8; N], _message_size: u32) -> [u8; 32]
 
 example:
 
-```rust
-fn main() {
-    let x = [163, 117, 178, 149]; // some random bytes
-    let message_size = 4;
-    let hash = std::hash::keccak256(x, message_size);
-}
-```
+#include_code keccak256 test_programs/execution_success/keccak256/src/main.nr rust
 
 <BlackBoxInfo />
 
@@ -116,13 +105,7 @@ fn hash_1(input: [Field; 1]) -> Field
 
 example:
 
-```rust
-fn main()
-{
-  let hash_2 = std::hash::poseidon::bn254::hash_2([1, 2]);
-  assert(hash2 == 0x115cc0f5e7d690413df64c6b9662e9cf2a3617f2743245519e19607a4417189a);
-}
-```
+#include_code poseidon test_programs/execution_success/poseidon_bn254_hash/src/main.nr rust
 
 ## mimc_bn254 and mimc
 
diff --git a/test_programs/execution_success/keccak256/src/main.nr b/test_programs/execution_success/keccak256/src/main.nr
index ff2167694d6..eb401fe614c 100644
--- a/test_programs/execution_success/keccak256/src/main.nr
+++ b/test_programs/execution_success/keccak256/src/main.nr
@@ -1,5 +1,4 @@
-// Keccak256 example
-//
+// docs:start:keccak256
 use dep::std;
 
 fn main(x: Field, result: [u8; 32]) {
@@ -7,7 +6,8 @@ fn main(x: Field, result: [u8; 32]) {
     // The padding is taken care of by the program
     let digest = std::hash::keccak256([x as u8], 1);
     assert(digest == result);
-    //#1399: variable meesage size
+
+    //#1399: variable message size
     let message_size = 4;
     let hash_a = std::hash::keccak256([1, 2, 3, 4], message_size);
     let hash_b = std::hash::keccak256([1, 2, 3, 4, 0, 0, 0, 0], message_size);
@@ -19,3 +19,4 @@ fn main(x: Field, result: [u8; 32]) {
 
     assert(hash_a != hash_c);
 }
+// docs:end:keccak256
diff --git a/test_programs/execution_success/pedersen_commitment/Nargo.toml b/test_programs/execution_success/pedersen_commitment/Nargo.toml
new file mode 100644
index 00000000000..e257ce252d8
--- /dev/null
+++ b/test_programs/execution_success/pedersen_commitment/Nargo.toml
@@ -0,0 +1,6 @@
+[package]
+name = "pedersen_commitment"
+type = "bin"
+authors = [""]
+
+[dependencies]
diff --git a/test_programs/execution_success/pedersen_commitment/Prover.toml b/test_programs/execution_success/pedersen_commitment/Prover.toml
new file mode 100644
index 00000000000..931b121fa6a
--- /dev/null
+++ b/test_programs/execution_success/pedersen_commitment/Prover.toml
@@ -0,0 +1,4 @@
+x = "0"
+y = "1"
+
+expected_hash = "0x0d98561fb02ca04d00801dfdc118b2a24cea0351963587712a28d368041370e1"
diff --git a/test_programs/execution_success/pedersen_commitment/src/main.nr b/test_programs/execution_success/pedersen_commitment/src/main.nr
new file mode 100644
index 00000000000..83cbe20851d
--- /dev/null
+++ b/test_programs/execution_success/pedersen_commitment/src/main.nr
@@ -0,0 +1,10 @@
+// docs:start:pedersen-commitment
+use dep::std;
+
+fn main(x: Field, y: Field, expected_commitment: std::hash::PedersenPoint) {
+    let commitment = std::hash::pedersen_commitment([x, y]);
+    assert_eq(commitment.x, expected_commitment.x);
+    assert_eq(commitment.y, expected_commitment.y);
+}
+// docs:end:pedersen-commitment
+
diff --git a/test_programs/execution_success/poseidon_bn254_hash/src/main.nr b/test_programs/execution_success/poseidon_bn254_hash/src/main.nr
index 3d30ebad279..e742a440d1c 100644
--- a/test_programs/execution_success/poseidon_bn254_hash/src/main.nr
+++ b/test_programs/execution_success/poseidon_bn254_hash/src/main.nr
@@ -1,3 +1,4 @@
+// docs:start:poseidon
 use dep::std::hash::poseidon;
 
 fn main(x1: [Field; 2], y1: pub Field, x2: [Field; 4], y2: pub Field) {
@@ -7,3 +8,4 @@ fn main(x1: [Field; 2], y1: pub Field, x2: [Field; 4], y2: pub Field) {
     let hash2 = poseidon::bn254::hash_4(x2);
     assert(hash2 == y2);
 }
+// docs:end:poseidon

From 7ab26c179b6991cf6b4104dea3e4e0bc12d6174b Mon Sep 17 00:00:00 2001
From: Tom French <tom@tomfren.ch>
Date: Thu, 25 Jan 2024 14:45:46 +0000
Subject: [PATCH 8/8] chore: fix `Prover.toml` for `pedersen_commitment`

---
 .../execution_success/pedersen_commitment/Prover.toml         | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test_programs/execution_success/pedersen_commitment/Prover.toml b/test_programs/execution_success/pedersen_commitment/Prover.toml
index 931b121fa6a..6ad8c6bca57 100644
--- a/test_programs/execution_success/pedersen_commitment/Prover.toml
+++ b/test_programs/execution_success/pedersen_commitment/Prover.toml
@@ -1,4 +1,6 @@
 x = "0"
 y = "1"
 
-expected_hash = "0x0d98561fb02ca04d00801dfdc118b2a24cea0351963587712a28d368041370e1"
+[expected_commitment]
+x = "0x054aa86a73cb8a34525e5bbed6e43ba1198e860f5f3950268f71df4591bde402"
+y = "0x209dcfbf2cfb57f9f6046f44d71ac6faf87254afc7407c04eb621a6287cac126"