Collect syntax kinds directly from DSL v2 and isolate parser genera…

…tion logic (NomicFoundation#991)

Work towards NomicFoundation#638 

Mainly:
- We bundle `quote`ing or any (old) parser codegen now in the new
`generator/src/parser/codegen` module
- To expand the `kinds.rs.jinja2` we use a new, dedicated `KindsModel`
to disentangle, limiting the workarounds like removing the built-in
labels only after visiting the grammar
- simplified collecting the `ParserModel.referenced_versions` (it's de
facto `Language::collect_breaking_changes` but with a caveat; I'll
submit a separate PR for that since it's not 100% obvious and would
change how we define `ParserModel`).
- and some minor fixes like `pub(super)` visibility or updated
referenced issues.

Now, the remaining work will be to actually revamp how we visit the
resulting grammar.

crates/codegen/language/definition/src/model/item.rs

@@ -39,4 +39,24 @@ impl Item {
             Item::Fragment { item } => &item.name,
         }
     }
+
+    /// Whether the language item corresponds to a dedicated terminal kind.
+    pub fn is_terminal(&self) -> bool {
+        // NOTE: `Item::Fragment` is inlined.
+        matches!(
+            self,
+            Item::Trivia { .. } | Item::Keyword { .. } | Item::Token { .. }
+        )
+    }
+
+    pub fn is_nonterminal(&self) -> bool {
+        matches!(
+            self,
+            Item::Struct { .. }
+                | Item::Enum { .. }
+                | Item::Repeated { .. }
+                | Item::Separated { .. }
+                | Item::Precedence { .. }
+        )
+    }
 }

crates/codegen/language/definition/src/model/utils/version_specifier.rs

@@ -20,4 +20,16 @@ impl VersionSpecifier {
             VersionSpecifier::Range { from, till } => from <= version && version < till,
         }
     }
+
+    /// Returns an iterator over the versions specified as the upper and lower bound.
+    pub fn versions(&self) -> impl Iterator<Item = &Version> {
+        match self {
+            VersionSpecifier::Never => [None, None],
+            VersionSpecifier::From { from } => [Some(from), None],
+            VersionSpecifier::Till { till } => [None, Some(till)],
+            VersionSpecifier::Range { from, till } => [Some(from), Some(till)],
+        }
+        .into_iter()
+        .flatten()
+    }
 }

crates/codegen/runtime/cargo/src/runtime/kinds.rs.jinja2

@@ -22,7 +22,7 @@ pub enum NonterminalKind {
         Stub2,
         Stub3,
     {%- else -%}
-        {%- for variant in model.parser.nonterminal_kinds -%}
+        {%- for variant in model.kinds.nonterminal_kinds -%}
             {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #}
             {{ variant }},
         {%- endfor -%}
@@ -65,7 +65,7 @@ pub enum EdgeLabel {
         Stub2,
         Stub3,
     {%- else -%}
-        {% for variant in model.parser.labels -%}
+        {% for variant in model.kinds.labels -%}
             {{ variant | pascal_case }},
         {%- endfor -%}
     {%- endif -%}
@@ -98,7 +98,7 @@ pub enum TerminalKind {
         Stub2,
         Stub3,
     {%- else -%}
-        {%- for variant in model.parser.terminal_kinds -%}
+        {%- for variant in model.kinds.terminal_kinds -%}
             {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #}
             {{ variant }},
         {%- endfor -%}
@@ -112,7 +112,7 @@ impl metaslang_cst::TerminalKind for TerminalKind {
         {%- else -%}
             matches!(
                 self,
-                {%- for variant in model.parser.trivia_scanner_names -%}
+                {%- for variant in model.kinds.trivia_scanner_names -%}
                     | Self::{{ variant }}
                 {%- endfor -%}
             )
@@ -128,7 +128,7 @@ pub(crate) enum LexicalContext {
         Stub2,
         Stub3,
     {%- else -%}
-        {%- for context_name, _ in model.parser.scanner_contexts %}
+        {%- for context_name in model.kinds.lexical_contexts %}
             {{ context_name }},
         {%- endfor %}
     {%- endif -%}
@@ -143,7 +143,7 @@ pub(crate) trait IsLexicalContext {
 #[allow(non_snake_case)]
 pub(crate) mod LexicalContextType {
     {%- if not rendering_in_stubs -%}
-        {%- for context_name, _ in model.parser.scanner_contexts %}
+        {%- for context_name in model.kinds.lexical_contexts %}
             pub struct {{ context_name }};
 
             impl super::IsLexicalContext for {{ context_name }} {

crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2

@@ -228,7 +228,7 @@ impl Lexer for Language {
                             if kw_scan == KeywordScan::Absent {
                                 input.set_position(save);
 
-                                // TODO(#638): Don't allocate a string here
+                                // TODO(#1001): Don't allocate a string here
                                 let ident_value = input.content(save.utf8..furthest_position.utf8);
 
                                 for keyword_compound_scanner in [

crates/codegen/runtime/generator/src/kinds.rs

@@ -0,0 +1,98 @@
+use std::collections::BTreeSet;
+
+use codegen_language_definition::model::{self, Identifier, Item};
+use serde::Serialize;
+
+#[derive(Default, Serialize)]
+pub struct KindsModel {
+    /// Defines the `NonterminalKind` enum variants.
+    nonterminal_kinds: BTreeSet<Identifier>,
+    /// Defines the `TerminalKind` enum variants.
+    terminal_kinds: BTreeSet<Identifier>,
+    /// Defines `TerminalKind::is_trivia` method.
+    trivia_scanner_names: BTreeSet<Identifier>,
+    /// Defines `EdgeLabel` enum variants.
+    labels: BTreeSet<Identifier>,
+    // Defines the `LexicalContext(Type)` enum and type-level variants.
+    lexical_contexts: BTreeSet<Identifier>,
+}
+
+impl KindsModel {
+    pub fn create(language: &model::Language) -> Self {
+        let terminal_kinds = language
+            .items()
+            .filter(|item| item.is_terminal() && !matches!(item, Item::Fragment { .. }))
+            .map(|item| item.name().clone())
+            .collect();
+
+        let mut nonterminal_kinds = BTreeSet::default();
+        for item in language.items() {
+            match item {
+                Item::Struct { item } => {
+                    nonterminal_kinds.insert(item.name.clone());
+                }
+                Item::Enum { item } => {
+                    nonterminal_kinds.insert(item.name.clone());
+                }
+                Item::Repeated { item } => {
+                    nonterminal_kinds.insert(item.name.clone());
+                }
+                Item::Separated { item } => {
+                    nonterminal_kinds.insert(item.name.clone());
+                }
+                Item::Precedence { item } => {
+                    nonterminal_kinds.insert(item.name.clone());
+                    for op in &item.precedence_expressions {
+                        nonterminal_kinds.insert(op.name.clone());
+                    }
+                }
+                // Terminals
+                _ => {}
+            }
+        }
+
+        let trivia_scanner_names = language
+            .items()
+            .filter_map(|item| match item {
+                Item::Trivia { item } => Some(item.name.clone()),
+                _ => None,
+            })
+            .collect();
+
+        let mut labels = BTreeSet::default();
+        for item in language.items() {
+            match item {
+                Item::Struct { item } => {
+                    for field_name in item.fields.keys() {
+                        labels.insert(field_name.clone());
+                    }
+                }
+                Item::Precedence { item } => {
+                    for item in &item.precedence_expressions {
+                        for item in &item.operators {
+                            for field_name in item.fields.keys() {
+                                labels.insert(field_name.clone());
+                            }
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        let lexical_contexts: BTreeSet<_> = language
+            .topics()
+            .filter_map(|t| t.lexical_context.as_ref())
+            .cloned()
+            .chain(std::iter::once(Identifier::from("Default")))
+            .collect();
+
+        KindsModel {
+            nonterminal_kinds,
+            terminal_kinds,
+            trivia_scanner_names,
+            labels,
+            lexical_contexts,
+        }
+    }
+}

crates/codegen/runtime/generator/src/lib.rs

@@ -10,6 +10,7 @@ use serde::Serialize;
 use crate::model::RuntimeModel;
 
 mod ast;
+mod kinds;
 mod model;
 mod parser;
 

crates/codegen/runtime/generator/src/model.rs

@@ -6,6 +6,7 @@ use semver::Version;
 use serde::Serialize;
 
 use crate::ast::AstModel;
+use crate::kinds::KindsModel;
 use crate::parser::ParserModel;
 
 #[derive(Default, Serialize)]
@@ -14,6 +15,7 @@ pub struct RuntimeModel {
     all_versions: BTreeSet<Version>,
     parser: ParserModel,
     ast: AstModel,
+    kinds: KindsModel,
 }
 
 impl RuntimeModel {
@@ -22,6 +24,7 @@ impl RuntimeModel {
             all_versions: language.versions.iter().cloned().collect(),
             ast: AstModel::create(language),
             parser: ParserModel::from_language(language),
+            kinds: KindsModel::create(language),
         }
     }
 }