made some progress but tests arent passing :/

deps/std/flags.ts

@@ -1 +1 @@
-export { parse } from "https://deno.land/std@0.110.0/flags/mod.ts";
+export { parse } from "https://deno.land/std@0.114.0/flags/mod.ts";

deps/std/fs.ts

@@ -1 +1 @@
-export { exists, expandGlob } from "https://deno.land/std@0.110.0/fs/mod.ts";
+export { exists, expandGlob } from "https://deno.land/std@0.114.0/fs/mod.ts";

deps/std/path.ts

@@ -5,4 +5,4 @@ export {
   join,
   normalize,
   parse,
-} from "https://deno.land/std@0.110.0/path/mod.ts";
+} from "https://deno.land/std@0.114.0/path/mod.ts";

deps/std/testing.ts

@@ -2,4 +2,4 @@ export {
   assert,
   assertEquals,
   assertThrows,
-} from "https://deno.land/std@0.110.0/testing/asserts.ts";
+} from "https://deno.land/std@0.114.0/testing/asserts.ts";

lib/text_builder/code_block/mod.ts

@@ -0,0 +1,2 @@
+export { CodeBlock } from "./code_block.ts";
+export { CodeLine } from "./code_line.ts";

lib/tokenize/lexicon.test.ts

@@ -0,0 +1,12 @@
+import { assertThrows } from "../../deps/std/testing.ts";
+
+import { LEXICON } from "./lexicon.ts";
+
+Deno.test("LEXICON is a frozen map", () => {
+  // @ts-expect-error: set throws intentionally
+  assertThrows(() => LEXICON.set("a", "b"));
+  // @ts-expect-error: delete throws intentionally
+  assertThrows(() => LEXICON.delete("a"));
+  // @ts-expect-error: clear throws intentionally
+  assertThrows(() => LEXICON.clear("a"));
+});

lib/tokenize/lexicon.ts

@@ -1,4 +1,3 @@
-// https://github.com/EthanThatOneKid/fart/blob/main/lib/consts/lexicon.ts
 export enum Lexicon {
   Identifier,
   StructOpener,
@@ -8,8 +7,10 @@ export enum Lexicon {
   TypeDefiner,
   PropertyDefiner,
   PropertyOptionalMarker,
+  PropertyOptionalDefiner,
   Modifier,
   TextWrapper,
+  TextLiteral,
   Comment,
   CommentOpener,
   CommentCloser,
@@ -18,19 +19,32 @@ export enum Lexicon {
 }
 
 export const LEXICON = new Map<Lexicon, string | string[] | null>([
-  [Lexicon.Identifier,  null],
+  [Lexicon.Identifier, null],
   [Lexicon.StructOpener, "{"],
   [Lexicon.StructCloser, "}"],
   [Lexicon.TupleOpener, "("],
   [Lexicon.TupleCloser, ")"],
   [Lexicon.TypeDefiner, ["type", "struct", "interface"]],
   [Lexicon.PropertyDefiner, ":"],
   [Lexicon.PropertyOptionalMarker, "?"],
-  [Lexicon.Modifier, ["%", "mod"]],
+  [Lexicon.PropertyOptionalDefiner, "?:"],
+  [Lexicon.Modifier, "%"],
   [Lexicon.TextWrapper, ['"', "'", "`"]],
+  [Lexicon.TextLiteral, null],
   [Lexicon.Comment, [";", "//"]],
   [Lexicon.CommentOpener, "/*"],
   [Lexicon.CommentCloser, "*/"],
   [Lexicon.Unknown, null],
   [Lexicon.EOF, null],
+]);
+
+// freezing LEXICON map into place, courtesy of https://stackoverflow.com/a/35776333
+LEXICON.set = function (key) {
+  throw new Error("Can't add property " + key + ", map is not extensible");
+};
+LEXICON.delete = function (key) {
+  throw new Error("Can't delete property " + key + ", map is frozen");
+};
+LEXICON.clear = function () {
+  throw new Error("Can't clear map, map is frozen");
 };

lib/tokenize/t.test.ts

@@ -0,0 +1,7 @@
+import { assertEquals } from "../../deps/std/testing.ts";
+
+import { T } from "./t.ts";
+
+Deno.test("TODO", () => {
+  assertEquals(T, T);
+});

lib/tokenize/t.ts

@@ -0,0 +1,77 @@
+// This file simply exports an object which contains lightweight
+// functions for creating Token instances with fewer keystrokes;
+// used primarily for testing-purposes.
+
+import { LEXICON, Lexicon } from "./lexicon.ts";
+import { Token } from "./token.ts";
+
+type SimpleTokenMaker = (line: number, col: number) => Token;
+type SpecialTokenMaker = (raw: string, line: number, col: number) => Token;
+
+interface LexiconAliasLayer {
+  /** `___` — identifier */
+  id: SpecialTokenMaker;
+  /** `{` — struct opener */
+  nest: SimpleTokenMaker;
+  /** `}` — struct closer */
+  denest: SimpleTokenMaker;
+  /** `(` — tuple opener */
+  open_tuple: SimpleTokenMaker;
+  /** `)` — tuple closer */
+  close_tuple: SimpleTokenMaker;
+  /** `type` — type definer */
+  type: SimpleTokenMaker;
+  /** `?` — optional marker */
+  optional: SimpleTokenMaker;
+  /** `:` — required setter */
+  setter_1: SimpleTokenMaker;
+  /** `?:` — optional setter */
+  setter_2: SimpleTokenMaker;
+  /** `%` — modifier */
+  mod: SimpleTokenMaker;
+  /** `"___"` — text literal (do not include quotes) */
+  text_1: SpecialTokenMaker;
+  /** `'___'` — text literal (do not include quotes) */
+  text_2: SpecialTokenMaker;
+  /** ```
+   * `___`
+   * ``` — text literal (do not include quotes) */
+  text_3: SpecialTokenMaker;
+  /** `;___` —  comment (include semicolon) */
+  comment: SpecialTokenMaker;
+  /** unknown */
+  unknown: SpecialTokenMaker;
+}
+
+const makeSpecialToken: SpecialTokenMaker = (raw, line, col) =>
+  new Token(raw, line, col);
+
+const NEST = LEXICON.get(Lexicon.StructOpener) as string;
+const DENEST = LEXICON.get(Lexicon.StructCloser) as string;
+const OPEN_TUPLE = LEXICON.get(Lexicon.TupleOpener) as string;
+const CLOSE_TUPLE = LEXICON.get(Lexicon.TupleCloser) as string;
+const TYPE = (LEXICON.get(Lexicon.TypeDefiner) as [string])[0];
+const OPTIONAL = LEXICON.get(Lexicon.PropertyOptionalMarker) as string;
+const SETTER_1 = LEXICON.get(Lexicon.PropertyDefiner) as string;
+const SETTER_2 = LEXICON.get(Lexicon.PropertyOptionalDefiner) as string;
+const MODIFIER = LEXICON.get(Lexicon.Modifier) as string;
+
+export const T: LexiconAliasLayer = {
+  id: makeSpecialToken,
+  nest: (line, col) => new Token(NEST, line, col),
+  denest: (line, col) => new Token(DENEST, line, col),
+  open_tuple: (line, col) => new Token(OPEN_TUPLE, line, col),
+  close_tuple: (line, col) => new Token(CLOSE_TUPLE, line, col),
+  type: (line, col) => new Token(TYPE, line, col),
+  optional: (line, col) => new Token(OPTIONAL, line, col),
+  setter_1: (line, col) => new Token(SETTER_1, line, col),
+  setter_2: (line, col) => new Token(SETTER_2, line, col),
+  mod: (line, col) => new Token(MODIFIER, line, col),
+  text_1: (raw, line, col) => new Token(`"${raw}"`, line, col),
+  text_2: (raw, line, col) => new Token(`'${raw}'`, line, col),
+  text_3: (raw, line, col) => new Token(`\`${raw}\``, line, col),
+  comment: makeSpecialToken,
+  unknown: makeSpecialToken,
+} as const;
+
+export default T;

lib/tokenize/token.test.ts

@@ -0,0 +1 @@
+// TODO: Make tests for Token class and getKindOf static function.

lib/tokenize/token.ts

@@ -1,21 +1,47 @@
-import { LEXICON, Lexicon } from "./lexicon.ts";
+import { Lexicon } from "./lexicon.ts";
+import {
+  checkIsIdentifier,
+  checkIsTextLiteral,
+  findInLexicon,
+} from "./utils.ts";
 
 export class Token {
   public kind: Lexicon | null = null;
-  public line: number = -1;
-  public column: number = -1;
 
   constructor(
     private raw: string,
-    line: number,
-    column: number,
+    public line = -1,
+    public column = -1,
     noCheck = false,
   ) {
-    this.line = line;
-    this.column = column;
     this.kind = noCheck ? Lexicon.Identifier : Token.getKindOf(raw);
   }
 
-  // https://github.com/EthanThatOneKid/fart/blob/c43f2333458b2cbc40d167610d87e2a2e3f89885/lib/tokenize/token.ts?_pjax=%23js-repo-pjax-container%2C%20div%5Bitemtype%3D%22http%3A%2F%2Fschema.org%2FSoftwareSourceCode%22%5D%20main%2C%20%5Bdata-pjax-container%5D#L48
-  static getKindOf(raw: string): Lexicon | null {}
+  is(kind: Lexicon | null): boolean {
+    return this.kind === kind;
+  }
+
+  toString() {
+    return this.value;
+  }
+
+  get value(): string {
+    switch (this.kind) {
+      case Lexicon.TextLiteral: {
+        // strips expected text markers from beginning and end of input string
+        return this.value.slice(1, this.value.length - 1);
+      }
+      default: {
+        return this.raw;
+      }
+    }
+  }
+
+  static getKindOf(raw: string): Lexicon {
+    const matchingKind = findInLexicon(raw);
+    if (matchingKind !== null) return matchingKind;
+    if (checkIsIdentifier(raw)) return Lexicon.Identifier;
+    if (checkIsTextLiteral(raw)) return Lexicon.TextLiteral;
+    return Lexicon.Unknown;
+  }
 }

lib/tokenize/utils.ts

@@ -0,0 +1,18 @@
+import { LEXICON, Lexicon } from "./lexicon.ts";
+
+export const findInLexicon = (raw: string): Lexicon | null => {
+  for (const [kind, value] of LEXICON) {
+    if ((Array.isArray(value) && value.includes(raw) || (raw === value))) {
+      return kind;
+    }
+  }
+  return null;
+};
+
+export const checkIsIdentifier = (candidate: string): boolean =>
+  /^[a-zA-Z_$]\.*[a-zA-Z_$0-9]*$/g.test(candidate);
+
+export const checkIsTextLiteral = (candidate: string): boolean =>
+  /^\`(.*?)\`$/g.test(candidate) ||
+  /^\'(.*?)\'$/g.test(candidate) ||
+  /^\"(.*?)\"$/g.test(candidate);