Add Tree Query Language and Engine (#753)

This pull request adds the CST Tree Query Language. It is a long way from being optimised, and in particular it should not need to allocate memory whilst backtracking, which it currently does. It is built with understandability as the top priority. This also includes some other changes that are not quite drive-by e.g. I rationalised the runtime crate structure and file naming to make more sense given I was adding a structural sub-module.
NomicFoundation · Feb 1, 2024 · b35c763 · b35c763
1 parent 7119e5c
commit b35c763
Show file tree

Hide file tree

Showing 69 changed files with 2,357 additions and 151 deletions.
diff --git a/.changeset/curvy-donkeys-shout.md b/.changeset/curvy-donkeys-shout.md
@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": minor
+---
+
+Add tree query implementation as `Query::parse` and `Cursor::query`
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -76,6 +76,7 @@ markdown = { version = "0.3.0" }
 napi = { version = "2.14.2", features = ["compat-mode", "napi8", "serde-json"] }
 napi-build = { version = "2.1.0" }
 napi-derive = { version = "2.14.6" }
+nom = { version = "7.1.3" }
 once_cell = { version = "1.19.0" }
 owo-colors = { version = "3.5.0", features = ["supports-colors"] }
 proc-macro2 = { version = "1.0.53" }
@@ -87,8 +88,9 @@ semver = { version = "1.0.17", features = ["serde"] }
 serde = { version = "1.0.158", features = ["derive", "rc"] }
 serde_json = { version = "1.0.94", features = ["preserve_order"] }
 similar-asserts = { version = "1.4.2" }
-strum = { version = "0.24.0" }
-strum_macros = { version = "0.24.0" }
+stack-graphs = { version = "0.12.0" }
+strum = { version = "0.25.0" }
+strum_macros = { version = "0.25.3" }
 syn = { version = "2.0.29", features = [
     "fold",
     "full",

diff --git a/crates/codegen/parser/generator/src/rust_generator.rs b/crates/codegen/parser/generator/src/rust_generator.rs
@@ -80,8 +80,8 @@ impl RustGenerator {
                 Context {
                     ast_model: AstModel::create(language),
                 },
-                runtime_dir.join("napi/templates/ast_selectors.rs.jinja2"),
-                output_dir.join("napi/napi_ast_selectors.rs"),
+                runtime_dir.join("napi_interface/templates/ast_selectors.rs.jinja2"),
+                output_dir.join("napi_interface/ast_selectors.rs"),
             )?;
         }
 
@@ -115,13 +115,13 @@ impl RustGenerator {
             )?;
         }
 
-        {
-            #[derive(Serialize)]
-            struct Context {}
-            codegen.render(
-                Context {},
-                runtime_dir.join("templates/mod.rs.jinja2"),
-                output_dir.join("mod.rs"),
+        for (src_file, destination_file) in &[
+            ("query/mod_for_destination.rs", "query/mod.rs"),
+            ("mod_for_destination.rs", "mod.rs"),
+        ] {
+            codegen.copy_file(
+                runtime_dir.join(src_file),
+                output_dir.join(destination_file),
             )?;
         }
 
@@ -131,25 +131,28 @@ impl RustGenerator {
             "lexer.rs",
             "parse_error.rs",
             "parse_output.rs",
+            "query/engine.rs",
+            "query/model.rs",
+            "query/parser.rs",
             "text_index.rs",
-            "napi/napi_cst.rs",
-            "napi/napi_cursor.rs",
-            "napi/napi_parse_error.rs",
-            "napi/napi_parse_output.rs",
-            "napi/napi_text_index.rs",
-            "napi/mod.rs",
-            "support/mod.rs",
-            "support/context.rs",
-            "support/parser_function.rs",
-            "support/optional_helper.rs",
-            "support/sequence_helper.rs",
-            "support/repetition_helper.rs",
-            "support/choice_helper.rs",
-            "support/precedence_helper.rs",
-            "support/parser_result.rs",
-            "support/recovery.rs",
-            "support/separated_helper.rs",
-            "support/scanner_macros.rs",
+            "napi_interface/cst.rs",
+            "napi_interface/cursor.rs",
+            "napi_interface/parse_error.rs",
+            "napi_interface/parse_output.rs",
+            "napi_interface/text_index.rs",
+            "napi_interface/mod.rs",
+            "parser_support/mod.rs",
+            "parser_support/context.rs",
+            "parser_support/parser_function.rs",
+            "parser_support/optional_helper.rs",
+            "parser_support/sequence_helper.rs",
+            "parser_support/repetition_helper.rs",
+            "parser_support/choice_helper.rs",
+            "parser_support/precedence_helper.rs",
+            "parser_support/parser_result.rs",
+            "parser_support/recovery.rs",
+            "parser_support/separated_helper.rs",
+            "parser_support/scanner_macros.rs",
         ] {
             codegen.copy_file(runtime_dir.join(file), output_dir.join(file))?;
         }

diff --git a/crates/codegen/parser/generator/src/typescript_generator.rs b/crates/codegen/parser/generator/src/typescript_generator.rs
@@ -26,7 +26,7 @@ impl TypeScriptGenerator {
                 Context {
                     ast_model: AstModel::create(language),
                 },
-                runtime_dir.join("napi/templates/ast_types.ts.jinja2"),
+                runtime_dir.join("napi_interface/templates/ast_types.ts.jinja2"),
                 output_dir.join("src/ast/generated/ast_types.ts"),
             )?;
         }

diff --git a/crates/codegen/parser/runtime/Cargo.toml b/crates/codegen/parser/runtime/Cargo.toml
@@ -15,6 +15,7 @@ napi-build = { workspace = true, optional = true }
 ariadne = { workspace = true }
 napi = { workspace = true, optional = true }
 napi-derive = { workspace = true, optional = true }
+nom = { workspace = true }
 serde = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }

diff --git a/crates/codegen/parser/runtime/src/kinds.rs b/crates/codegen/parser/runtime/src/kinds.rs
@@ -17,7 +17,11 @@ use napi_derive::napi;
 #[cfg_attr(not(feature = "slang_napi_interfaces"), derive(Clone, Copy))]
 pub enum TokenKind {
     SKIPPED,
-    // Expanded by the template engine
+    // Used for testing this crate, this is generated in the client code
+    Identifier,
+    Token1,
+    Token2,
+    Token3,
 }
 
 #[derive(
@@ -37,7 +41,10 @@ pub enum TokenKind {
 pub enum RuleKind {
     LeadingTrivia,
     TrailingTrivia,
-    // Expanded by the template engine
+    // Used for testing this crate, this is generated in the client code
+    Rule1,
+    Rule2,
+    Rule3,
 }
 
 impl RuleKind {
@@ -69,8 +76,10 @@ pub enum FieldName {
     Operand,
     LeftOperand,
     RightOperand,
-    // Generated
-    XXX,
+    // Used for testing this crate, this is generated in the client code
+    Name1,
+    Name2,
+    Name3,
 }
 
 /// The lexical context of the scanner.

diff --git a/crates/codegen/parser/runtime/src/lexer.rs b/crates/codegen/parser/runtime/src/lexer.rs
@@ -1,6 +1,6 @@
 use crate::cst::{self, NamedNode};
 use crate::kinds::{IsLexicalContext, TokenKind};
-use crate::support::{ParserContext, ParserResult};
+use crate::parser_support::{ParserContext, ParserResult};
 
 /// Whether a keyword has been scanned and if so, whether it is reserved (unusable as an identifier)
 /// or not.