More tidies and a tokenisation diagram for the blog post

guardian · Oct 24, 2024 · a5737f8 · a5737f8
1 parent b691194
commit a5737f8
Show file tree

Hide file tree

Showing 8 changed files with 150 additions and 86 deletions.
diff --git a/prosemirror-client/src/cqlInput/editor/debug.ts b/prosemirror-client/src/cqlInput/editor/debug.ts
@@ -1,5 +1,8 @@
 import { Node } from "prosemirror-model";
 import { Mapping } from "prosemirror-transform";
+import { Token } from "../../lang/token";
+
+// Debugging and visualisation utilities.
 
 /**
  * Utility function to log node structure to console.
@@ -19,6 +22,72 @@ export const logNode = (doc: Node) => {
   });
 };
 
+export const getDebugTokenHTML = (tokens: Token[]) => {
+  let html = `
+    <div class="CqlDebug__queryDiagram CqlDebug__queryDiagramToken">
+      <div class="CqlDebug__queryDiagramLabel">
+        <div>Lexeme</div>
+        <div>Literal</div>
+      </div>
+      <div class="CqlDebug__queryDiagramContent">`;
+  tokens.forEach((token, index) => {
+    html += `${Array(Math.max(1, token.lexeme.length))
+      .fill(undefined)
+      .map((_, index) => {
+        const lexemeChar = token.lexeme[index];
+        const literalOffset =
+          token.literal?.length === token.lexeme.length ? 0 : 1;
+        const literalChar = token.literal?.[index - literalOffset];
+        return `
+        <div class="CqlDebug__queryBox">
+          <div class="CqlDebug__queryIndex">${token.start + index}</div>
+          ${
+            lexemeChar !== undefined
+              ? `<div class="CqlDebug__queryChar">${lexemeChar}</div>`
+              : ""
+          }
+              ${
+                literalChar !== undefined
+                  ? `<div class="CqlDebug__queryChar CqlDebug__queryCharAlt">${literalChar}</div>`
+                  : ""
+              }
+          ${
+            index === 0
+              ? `<div class="CqlDebug__tokenLabel">${token.tokenType}</div>`
+              : ""
+          }
+        </div>`;
+      })
+      .join("")}
+      ${
+        tokens[index + 1]?.tokenType !== "EOF" && token.tokenType !== "EOF"
+          ? `<div class="CqlDebug__queryBox"><div class="CqlDebug__queryIndex">${
+              token.end + 1
+            }</div></div>`
+          : ""
+      }`;
+  });
+  html += "</div></div>";
+
+  return html;
+};
+
+export const getOriginalQueryHTML = (query: string) => `
+<div class="CqlDebug__queryDiagram">
+  <div class="CqlDebug__queryDiagramContent">
+  ${query
+    .split("")
+    .map(
+      (char, index) => `
+          <div class="CqlDebug__queryBox">
+              <div class="CqlDebug__queryIndex">${index}</div>
+              <div class="CqlDebug__queryChar">${char}</div>
+          </div>`
+    )
+    .join("")}
+    </div>
+</div>`;
+
 export const getDebugMappingHTML = (
   query: string,
   mapping: Mapping,
@@ -34,24 +103,13 @@ export const getDebugMappingHTML = (
       : [posInfo];
   });
 
-  const queryDiagram = `
-    <p>Original query:</p>
-    <div class="CqlDebug__queryDiagram">
-
-    ${query
-      .split("")
-      .map(
-        (char, index) => `
-            <div class="CqlDebug__queryBox">
-                <div class="CqlDebug__queryIndex">${index}</div>
-                <div class="CqlDebug__queryChar">${char}</div>
-            </div>`
-      )
-      .join("")}
-    </div>`;
-
   let nodeDiagram = `
-    <p>Maps to nodes: </p><div class="CqlDebug__nodeDiagram">`;
+    <div class="CqlDebug__queryDiagram CqlDebug__queryDiagramNode">
+    <div class="CqlDebug__queryDiagramLabel">
+        <div>Mapped query</div>
+        <div>Node text</div>
+      </div>
+      <div class="CqlDebug__nodeDiagram">`;
   const posMap: Record<string, { char?: string; node?: string }> = {};
   doc.nodesBetween(0, doc.content.size, (node, pos) => {
     const content =
@@ -87,21 +145,21 @@ export const getDebugMappingHTML = (
       ([pos, { char, node }]) =>
         `
                 <div class="CqlDebug__queryBox CqlDebug__queryBox--offset" data-pos="${pos}">
-                <div class="CqlDebug__queryIndex">${pos}</div>
+                    <div class="CqlDebug__queryIndex">${pos}</div>
                     ${(queryPosMap[pos] ?? []).map(
                       ({ char }) =>
                         `<div class="CqlDebug__originalChar">${char}</div>`
                     )}
 
-                    <div class="CqlDebug__queryIndex">${pos}</div>
+
                     ${
                       char?.length === 1
                         ? `<div class="CqlDebug__nodeChar">${char}</div>`
                         : ""
                     }
                     ${
                       node?.length
-                        ? `<div class="CqlDebug__node ${
+                        ? `<div class="CqlDebug__nodeLabel ${
                             node === "text" ? "CqlDebug__textNode" : ""
                           }">${node}</div>`
                         : ""
@@ -110,7 +168,11 @@ export const getDebugMappingHTML = (
     )
     .join("");
 
-  nodeDiagram += `</div>`;
+  nodeDiagram += `
+    </div>
+  </div>`;
+
+  return nodeDiagram;
 
-  return `<div class="CqlDebug__mapping">${queryDiagram}${nodeDiagram}</div>`;
+  //  return `<div class="CqlDebug__mapping">${queryDiagram}${nodeDiagram}</div>`;
 };
diff --git a/prosemirror-client/src/cqlInput/editor/plugin.ts b/prosemirror-client/src/cqlInput/editor/plugin.ts
@@ -26,7 +26,11 @@ import { QueryChangeEventDetail } from "../../types/dom";
 import { ErrorPopover } from "../ErrorPopover";
 import { MappedTypeaheadSuggestion } from "../../lang/types";
 import { CqlConfig } from "../CqlInput";
-import { getDebugMappingHTML } from "./debug";
+import {
+  getDebugMappingHTML,
+  getDebugTokenHTML,
+  getOriginalQueryHTML,
+} from "./debug";
 
 const cqlPluginKey = new PluginKey<PluginState>("cql-plugin");
 
@@ -153,7 +157,7 @@ export const createCqlPlugin = ({
               return;
             }
 
-            const $pos = view.state.doc.resolve(pos)
+            const $pos = view.state.doc.resolve(pos);
             const node = $pos.nodeAfter;
 
             if (!node) {
@@ -165,7 +169,7 @@ export const createCqlPlugin = ({
 
           const dom = document.createElement("chip-wrapper");
           const contentDOM = document.createElement("span");
-          contentDOM.classList.add("Cql__ChipWrapperContent")
+          contentDOM.classList.add("Cql__ChipWrapperContent");
           const polarityHandle = document.createElement("span");
           polarityHandle.classList.add("Cql__ChipWrapperPolarityHandle");
           polarityHandle.setAttribute("contentEditable", "false");
@@ -333,11 +337,14 @@ export const createCqlPlugin = ({
           }
 
           if (debugMappingContainer) {
-            debugMappingContainer.innerHTML = getDebugMappingHTML(
-              query,
-              mapping,
-              newDoc
-            );
+            debugMappingContainer.innerHTML = `
+              <p>Original query: </p>
+              ${getOriginalQueryHTML(query)}
+              <p>Tokenises to:</p>
+              ${getDebugTokenHTML(result.tokens)}
+              <p>Maps to nodes: </p>
+              ${getDebugMappingHTML(query, mapping, newDoc)}
+            `;
           }
 
           const userSelection = view.state.selection;

diff --git a/prosemirror-client/src/lang/parser.spec.ts b/prosemirror-client/src/lang/parser.spec.ts
@@ -10,12 +10,10 @@ import {
 } from "./ast";
 import {
   andToken,
-  colonToken,
   eofToken,
   leftParenToken,
   queryField,
   queryFieldKeyToken,
-  queryOutputModifierKeyToken,
   queryValueToken,
   quotedStringToken,
   rightParenToken,
@@ -152,24 +150,10 @@ describe("parser", () => {
     assertFailure(result, "unexpected ':'");
   });
 
-  it("should handle a colon with no query key after another query", () => {
-    const tokens = [
-      queryFieldKeyToken("tag", 5),
-      queryValueToken("news", 8),
-      colonToken(13),
-      eofToken(14),
-    ];
-    const result = new Parser(tokens).parse();
-    assertFailure(result, "unexpected ':'");
-  });
-
   it("should not crash on arbitrary tokens", () => {
     const tokens = [
       queryFieldKeyToken("tag"),
       queryValueToken("news"),
-      queryOutputModifierKeyToken("show-fields"),
-      queryValueToken("all"),
-      colonToken(13),
       andToken(1),
       leftParenToken(),
       quotedStringToken("sausages"),

diff --git a/prosemirror-client/src/lang/parser.ts b/prosemirror-client/src/lang/parser.ts
@@ -14,7 +14,7 @@ import {
   QueryStr,
 } from "./ast";
 import { TokenType } from "./token";
-import { either, err, mapError, ok, Result } from "../utils/result";
+import { either, err, ok, Result } from "../utils/result";
 
 class ParseError extends Error {
   constructor(public position: number, public message: string) {
@@ -47,8 +47,8 @@ export class Parser {
     }
   }
 
-  private startOfQueryField = [TokenType.QUERY_FIELD_KEY, TokenType.PLUS];
-  private startOfQueryValue = [TokenType.QUERY_VALUE, TokenType.COLON];
+  private startOfQueryField = [TokenType.QUERY_FIELD_KEY];
+  private startOfQueryValue = [TokenType.QUERY_VALUE];
 
   private query(): QueryBinary {
     if (this.startOfQueryValue.some((i) => i === this.peek().tokenType))
@@ -156,14 +156,7 @@ export class Parser {
       `Expected a search value, e.g. +tag:new`
     );
 
-    const value = mapError(() =>
-      this.safeConsume(
-        TokenType.COLON,
-        `Expected an ':' after the search key ${key.lexeme}`
-      )
-    )(maybeValue);
-
-    return either(value)(
+    return either(maybeValue)(
       () => createQueryField(key, undefined),
       (value: Token) => createQueryField(key, value)
     );
@@ -175,9 +168,6 @@ export class Parser {
    */
   private guardAgainstQueryField = (errorLocation: string) => {
     switch (this.peek().tokenType) {
-      case TokenType.PLUS: {
-        throw this.error(`You cannot put query fields ${errorLocation}"`);
-      }
       case TokenType.QUERY_FIELD_KEY: {
         const queryFieldNode = this.queryField();
         throw this.error(

diff --git a/prosemirror-client/src/lang/scanner.ts b/prosemirror-client/src/lang/scanner.ts
@@ -21,9 +21,6 @@ export class Scanner {
     );
   };
 
-  private isAtEnd = (offset: number = 0) =>
-    this.current + offset === this.program.length;
-
   private scanToken = () => {
     switch (this.advance()) {
       case "+":
@@ -55,7 +52,7 @@ export class Scanner {
     while (this.peek() != ":" && !isWhitespace(this.peek()) && !this.isAtEnd())
       this.advance();
 
-    if (this.current - this.start == 1) this.addToken(tokenType);
+    if (this.current - this.start === 1) this.addToken(tokenType);
     else {
       const key = this.program.substring(this.start + 1, this.current);
 
@@ -141,6 +138,9 @@ export class Scanner {
       ? "\u0000"
       : this.program[this.current + offset];
 
+  private isAtEnd = (offset: number = 0) =>
+    this.current + offset === this.program.length;
+
   private error = (line: number, message: String) =>
     this.report(line, "", message);
 

diff --git a/prosemirror-client/src/lang/testUtils.ts b/prosemirror-client/src/lang/testUtils.ts
@@ -9,8 +9,6 @@ export const andToken = (start: number = 0) =>
   new Token(TokenType.AND, "AND", "AND", start, start + 3);
 export const eofToken = (start: number) =>
   new Token(TokenType.EOF, "", undefined, start, start);
-export const colonToken = (start: number) =>
-  new Token(TokenType.COLON, "", undefined, start, start);
 export const unquotedStringToken = (str: string, start: number = 0) =>
   new Token(TokenType.STRING, str, str, start, start + str.length - 1);
 export const quotedStringToken = (str: string, start: number = 0) =>
@@ -23,14 +21,7 @@ export const queryFieldKeyToken = (str: string, start: number = 0) =>
     start,
     start + str.length
   );
-export const queryOutputModifierKeyToken = (str: string, start: number = 0) =>
-  new Token(
-    TokenType.QUERY_OUTPUT_MODIFIER_KEY,
-    `@${str}`,
-    str,
-    start,
-    start + str.length
-  );
+
 export const queryValueToken = (str: string, start: number = 0) =>
   new Token(TokenType.QUERY_VALUE, `:${str}`, str, start, start + str.length);
 

diff --git a/prosemirror-client/src/lang/token.ts b/prosemirror-client/src/lang/token.ts
@@ -2,16 +2,11 @@ export type TokenType = keyof typeof TokenType;
 
 export const TokenType = {
   // Single-character tokens.
-  PLUS: "PLUS",
-  COLON: "COLON",
-  AT: "AT",
   LEFT_BRACKET: "LEFT_BRACKET",
   RIGHT_BRACKET: "RIGHT_BRACKET",
 
   // Literals.
   STRING: "STRING",
-  NUMBER: "NUMBER",
-  QUERY_OUTPUT_MODIFIER_KEY: "QUERY_OUTPUT_MODIFIER_KEY",
   QUERY_FIELD_KEY: "QUERY_FIELD_KEY",
   QUERY_VALUE: "QUERY_VALUE",