compile_templates.py

import os
import re
import sys

# the dir of this file
__DIR__ = os.path.dirname(os.path.realpath(__file__))

sys.path.insert(0, os.path.join(__DIR__, "libs"))
os.chdir(__DIR__)

import triegex

# fmt: off
TOKENS_FILES = [
    "parser-token",
]
# fmt: on

OUTPUT_DIR = "compiled"

os.path.exists(OUTPUT_DIR) or os.mkdir(OUTPUT_DIR)

for tokens_file in TOKENS_FILES:
    regex_optimized = ""
    yaml_template = ""

    with open(tokens_file + ".txt", "r") as f:
        lines = [line.rstrip() for line in f]
        tokens = filter(bool, lines)  # remove empty lines
        regex_optimized = (
            triegex.Triegex(*map(re.escape, tokens))
            .to_regex()
            # some clean up
            .replace(r"\b", "")
            .replace(r"|~^(?#match nothing)", "")
        )

    with open(tokens_file + ".tpl.yaml", "r") as f:
        yaml_template = f.read()

    with open(os.path.join(OUTPUT_DIR, tokens_file + ".sublime-syntax"), "w") as f:
        # fmt: off
        compiled_template = (
            yaml_template
            .replace("{#tokens_file#}", tokens_file)
            .replace("{#regex_optimized#}", regex_optimized)
        )
        # fmt: on

        f.write(compiled_template)