First Files

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/lachs"
+    schedule:
+      interval: "daily"
+
+  - package-ecosystem: "cargo"
+    directory: "/lachs_derive"
+    schedule:
+      interval: "daily"

.github/workflows/ci.yml

@@ -0,0 +1,96 @@
+name: CI Checks
+
+on:
+  push:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  building:
+    name: Building
+    continue-on-error: ${{ matrix.experimental || false }}
+    strategy:
+      matrix:
+        # All generated code should be running on stable now
+        rust:
+          - stable
+          - nightly
+        include:
+          # Nightly is only for reference and allowed to fail
+          - rust: nightly
+            experimental: true
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.rust }}
+          override: true
+
+      - uses: Swatinem/rust-cache@v2
+
+      - run: cargo build --all
+
+      - run: cargo build --all --release
+
+  testing:
+    needs: building
+    name: Testing
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - run: cargo test
+
+  linting:
+    name: Linting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          components: clippy
+
+      - uses: Swatinem/rust-cache@v2
+
+      - run: cargo clippy --all-targets --workspace
+
+  formatting:
+    name: Formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          components: rustfmt
+
+      - uses: Swatinem/rust-cache@v2
+
+      - run: cargo fmt -- --check

.gitignore

@@ -18,4 +18,4 @@ Cargo.lock
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+#.idea/

Cargo.toml

@@ -0,0 +1,2 @@
+[workspace]
+members = ["lachs", "lachs_derive"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Louis Meyer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,72 @@
+# Lachs
+
+A tool to automatically generate a lexer based on a given enum.
+
+## Usage
+
+To generate a lexer from a given struct, just annotate it with [`token`]:
+
+```rust
+use lachs::token;
+
+#[token]
+pub enum Token {
+    #[terminal("+")]
+    Plus,
+    #[literal("[0-9]+")]
+    Integer
+}
+```
+
+As you can see, we also annotated the variants `Token::Plus` and `Token::Integer` with `#[terminal("+")]` and `#[literal("[0-9]+")]`, respectively.
+
+The helper `#[terminal(...)]` takes a string literal which has to match exactly to be lexed as the decorated token, while `#[literal(...)]` takes
+a regular expression to extract a matched sequence from the text.
+
+These helper macros get evaluated by `#[token]` and describe the two different kinds of tokens the lexer can understand:
+
+- terminals (without an own value)
+- literals (with an own value)
+
+Under the hood, the proc macro expands the struct to roughly the following:
+
+```rust
+pub enum Token {
+    Plus {
+        position: lachs::Span,
+    },
+    Integer {
+        value: String,
+        position: lachs::Span,
+    }
+}
+```
+
+Both, terminals and literals have a field named `position` to store the position in the originating text.
+Literals have an additional field `value` which stores the value which matched the passed regular expression.
+
+Additionally, the `Token` enum gets a function which lets you pass a string and get the result of the lexing back:
+
+```rust
+use lachs::token;
+
+#[token]
+pub enum Token {
+    #[terminal("+")]
+    Plus,
+    #[literal("[0-9]+")]
+    Integer
+}
+
+let result: Result<Vec<Token>, LexError> = Token::lex("2 + 2");
+```
+
+## Caveats
+
+The macro also generates an implementation of `PartialEq` for the decorated enum. However, this implementation _does **not**_ take the position into account.
+
+If you want to check whether two tokens are exactly the same, you can utilize the `Token::does_equal(...)` function.
+
+### Generated Stuff
+
+The macro generates additional structs for performing the actual lexing. These should not be touched, if possible. However, they can lead to name collisions.

lachs/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "lachs"
+version = "0.1.0"
+edition = "2021"
+authors = ["Louis Meyer (H1ghBre4k3r) <h1ghbe4k3r@dev.bre4k3r.de>"]
+license = "MIT"
+readme = "README.md"
+description = "Crate for automatically creating a lexer based on a given enum"
+repository = "https://github.com/H1ghBre4k3r/lachs"
+documentation = "https://docs.rs/lachs"
+
+[dependencies]
+colored = "2.1.0"
+lachs_derive = { version = "0.1.0", path = "../lachs_derive/" }
+regex = "1.11.1"

lachs/LICENSE

@@ -0,0 +1 @@
+../LICENSE

lachs/README.md

@@ -0,0 +1 @@
+../README.md

lachs/src/lib.rs

@@ -0,0 +1,90 @@
+#![doc = include_str!("../../README.md")]
+pub use colored;
+pub use lachs_derive::token;
+pub use regex;
+
+use colored::Colorize;
+
+#[derive(Default, Debug, Clone, Eq)]
+pub struct Span {
+    pub start: (usize, usize),
+    pub end: (usize, usize),
+    pub source: String,
+}
+
+impl Span {
+    pub fn to_string(&self, msg: impl ToString) -> String {
+        let Span { start, end, source } = self;
+        let line = start.0;
+        let lines = source.lines().collect::<Vec<_>>();
+        let prev_line = if line > 0 { lines[line - 1] } else { "" };
+        let line_str = lines[line];
+
+        // margin _before_ left border
+        let left_margin = format!("{}", end.0).len();
+        let left_margin_fill = vec![' '; left_margin].iter().collect::<String>();
+
+        // split right at the start of the error in the first line
+        let (left, right) = line_str.split_at(start.1);
+
+        // some case magic
+        let (left, right) = if start.0 != end.0 {
+            // if the error ranges over more than a single line, we can just mark rest of the line
+            // as an error
+            (left.to_string(), right.to_string().red().to_string())
+        } else {
+            // however, if the lines does not range beyond this line, we need to split at the end
+            // again
+            let (err_str, after_err) = right.split_at(end.1 - start.1);
+
+            // now, just color the error part red
+            (
+                left.to_string(),
+                format!("{err_str}{after_err}", err_str = err_str.to_string().red()),
+            )
+        };
+
+        // and concatentate both together
+        let line_str = format!("{left}{right}");
+
+        // padding between border and squiggles
+        let left_padding_fill = vec![' '; end.1 - 1].iter().collect::<String>();
+
+        // the error with the first line
+        let mut error_string = format!(
+            "{left_margin_fill} |\n{left_margin_fill} |{prev_line} \n{line} |{line_str}",
+            line = line + 1
+        );
+
+        // iterate over all lines of the error and make them shine red
+        ((start.0 + 1)..(end.0 + 1)).for_each(|line_number| {
+            error_string = format!(
+                "{error_string}\n{left_margin_fill} |{}",
+                lines[line_number].to_string().red()
+            );
+        });
+
+        // actually add error message at bottom
+        error_string = format!(
+            "{error_string}\n{} |{left_padding_fill}^--- {}\n{left_margin_fill} |",
+            end.0 + 2,
+            msg.to_string()
+        );
+
+        error_string
+    }
+
+    pub fn merge(&self, other: &Span) -> Span {
+        let Span { start, source, .. } = self.clone();
+        let Span { end, .. } = other.clone();
+
+        Span { start, end, source }
+    }
+}
+
+impl PartialEq<Span> for Span {
+    fn eq(&self, _other: &Span) -> bool {
+        // TODO: maybe this should not be the case...
+        true
+    }
+}

lachs_derive/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "lachs_derive"
+version = "0.1.0"
+edition = "2021"
+authors = ["Louis Meyer (H1ghBre4k3r) <h1ghbe4k3r@dev.bre4k3r.de>"]
+license = "MIT"
+readme = "README.md"
+description = "Derive macro for creating a lexer"
+repository = "https://github.com/H1ghBre4k3r/lachs"
+documentation = "https://docs.rs/lachs_derive"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+quote = "1.0.37"
+syn = { version = "2.0.76", features = ["full"] }

lachs_derive/LICENSE

@@ -0,0 +1 @@
+../LICENSE

lachs_derive/README.md

@@ -0,0 +1,9 @@
+# Lachs Derive
+
+> IMPORTANT: You are not supposed to use this crate directly.
+> The only way we guarantee a working usage is to directly use the `lachs` crate.
+
+This crate provides the implementation of a lexer generator in the form
+of the `#[token]` proc macro. Applying this macro to an enum will consume
+this enum and generate a new enum where each variant has the fields
+corresponding to the type of token this field represents.

lachs_derive/src/lib.rs

@@ -0,0 +1,13 @@
+#[doc = include_str!("../README.md")]
+mod token;
+
+use token::*;
+
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn token(_attr: TokenStream, input: TokenStream) -> TokenStream {
+    let ast = syn::parse(input).expect("#[token] currently only works for items!");
+
+    impl_token_macro(ast)
+}