Mirror rowan architecture in our CST (#502)

NomicFoundation · Jul 5, 2023 · c383238 · c383238
1 parent 560ad1b
commit c383238
Show file tree

Hide file tree

Showing 190 changed files with 22,690 additions and 35,941 deletions.
diff --git a/.changeset/five-cougars-lay.md b/.changeset/five-cougars-lay.md
@@ -0,0 +1,5 @@
+---
+"changelog": minor
+---
+
+Added error recovery i.e. a CST is _always_ produced, even if there are errors. The erroneous/skipped text is in the CST as a `TokenKind::SKIPPED` token.
diff --git a/.changeset/shy-carrots-love.md b/.changeset/shy-carrots-love.md
@@ -0,0 +1,5 @@
+---
+"changelog": minor
+---
+
+Use the Rowan model for the CST i.e. TokenNodes contain the string content, and RuleNodes contain only the combined _length_ of their children's text.
diff --git a/crates/codegen/ebnf/src/nodes.rs b/crates/codegen/ebnf/src/nodes.rs
@@ -24,11 +24,6 @@ pub enum EbnfNode {
         from: char,
         to: char,
     },
-    Repeat {
-        min: usize,
-        max: usize,
-        body: Box<EbnfNode>,
-    },
     Sequence {
         elements: Vec<EbnfNode>,
     },
@@ -83,14 +78,6 @@ impl EbnfNode {
         Self::Range { from, to }
     }
 
-    pub fn repeat(min: usize, max: usize, body: EbnfNode) -> Self {
-        Self::Repeat {
-            min,
-            max,
-            body: Box::new(body),
-        }
-    }
-
     pub fn sequence(elements: Vec<EbnfNode>) -> Self {
         Self::Sequence { elements }
     }

diff --git a/crates/codegen/ebnf/src/parser.rs b/crates/codegen/ebnf/src/parser.rs
@@ -44,14 +44,6 @@ impl GenerateEbnf for ParserDefinition {
                 return EbnfNode::production_ref(name.to_owned());
             }
 
-            ParserDefinition::Repeat {
-                min,
-                max,
-                expression,
-            } => {
-                return EbnfNode::repeat(*min, *max, expression.generate_ebnf());
-            }
-
             ParserDefinition::SeparatedBy {
                 expression,
                 separator,

diff --git a/crates/codegen/ebnf/src/scanner.rs b/crates/codegen/ebnf/src/scanner.rs
@@ -50,14 +50,6 @@ impl GenerateEbnf for ScannerDefinition {
                 return EbnfNode::production_ref(name.to_owned());
             }
 
-            ScannerDefinition::Repeat {
-                min,
-                max,
-                expression,
-            } => {
-                return EbnfNode::repeat(*min, *max, expression.generate_ebnf());
-            }
-
             ScannerDefinition::Sequence(elements) => {
                 return EbnfNode::sequence(
                     elements

diff --git a/crates/codegen/ebnf/src/serialization.rs b/crates/codegen/ebnf/src/serialization.rs
@@ -136,14 +136,6 @@ impl<'language> EbnfSerializer<'language> {
             EbnfNode::ProductionRef { name } => {
                 self.buffer.push_str(&self.display_name(name));
             }
-            EbnfNode::Repeat { min, max, body } => {
-                self.serialize_child_node(node, body);
-                self.buffer.push_str("{");
-                self.buffer.push_str(&min.to_string());
-                self.buffer.push_str(",");
-                self.buffer.push_str(&max.to_string());
-                self.buffer.push_str("}");
-            }
             EbnfNode::Sequence { elements } => {
                 for (i, element) in elements.into_iter().enumerate() {
                     if i > 0 {
@@ -207,10 +199,7 @@ fn precedence(node: &EbnfNode) -> u8 {
         EbnfNode::Not { .. } => 1,
 
         // Postfix
-        EbnfNode::OneOrMore { .. }
-        | EbnfNode::Optional { .. }
-        | EbnfNode::Repeat { .. }
-        | EbnfNode::ZeroOrMore { .. } => 2,
+        EbnfNode::OneOrMore { .. } | EbnfNode::Optional { .. } | EbnfNode::ZeroOrMore { .. } => 2,
 
         // Primary
         EbnfNode::BaseProduction

diff --git a/crates/codegen/schema/src/types/parser.rs b/crates/codegen/schema/src/types/parser.rs
@@ -36,13 +36,6 @@ pub enum ParserDefinition {
     #[schemars(title = "Reference Expression")]
     Reference(String),
 
-    #[schemars(title = "Repeat Expression")]
-    Repeat {
-        expression: ParserRef,
-        min: usize,
-        max: usize,
-    },
-
     #[schemars(title = "SeparatedBy Expression")]
     SeparatedBy {
         expression: ParserRef,

diff --git a/crates/codegen/schema/src/types/scanner.rs b/crates/codegen/schema/src/types/scanner.rs
@@ -37,13 +37,6 @@ pub enum ScannerDefinition {
     #[schemars(title = "Reference Expression")]
     Reference(String),
 
-    #[schemars(title = "Repeat Expression")]
-    Repeat {
-        expression: ScannerRef,
-        min: usize,
-        max: usize,
-    },
-
     #[schemars(title = "Sequence Expression")]
     Sequence(Vec<ScannerRef>),
 

diff --git a/crates/codegen/schema/src/validation/rules/empty_roots/mod.rs b/crates/codegen/schema/src/validation/rules/empty_roots/mod.rs
@@ -99,16 +99,6 @@ fn possible_empty_scanner(scanner: &ScannerRef) -> bool {
             return sequence.iter().all(possible_empty_scanner);
         }
 
-        ScannerDefinition::Repeat {
-            min, expression, ..
-        } => {
-            if *min == 0 {
-                return true;
-            }
-
-            return possible_empty_scanner(expression);
-        }
-
         ScannerDefinition::Not { .. }
         | ScannerDefinition::Range { .. }
         | ScannerDefinition::Reference(_)
@@ -138,16 +128,6 @@ fn possible_empty_parser(parser: &ParserRef) -> bool {
             return sequence.iter().all(possible_empty_parser);
         }
 
-        ParserDefinition::Repeat {
-            min, expression, ..
-        } => {
-            if *min == 0 {
-                return true;
-            }
-
-            return possible_empty_parser(expression);
-        }
-
         ParserDefinition::DelimitedBy { .. } | ParserDefinition::Reference(_) => {
             return false;
         }

diff --git a/crates/codegen/schema/src/validation/visitors/receivers.rs b/crates/codegen/schema/src/validation/visitors/receivers.rs
@@ -164,9 +164,6 @@ impl Receiver for ScannerDefinition {
             }
             ScannerDefinition::Range { .. } => {}
             ScannerDefinition::Reference(_) => {}
-            ScannerDefinition::Repeat { expression, .. } => {
-                expression.receive(visitor, language, location.field("repeat"), reporter);
-            }
             ScannerDefinition::Sequence(expressions) => {
                 let location = location.field("sequence");
                 for (i, expression) in expressions.iter().enumerate() {
@@ -237,10 +234,6 @@ impl Receiver for ParserDefinition {
                 expression.receive(visitor, language, location.field("optional"), reporter);
             }
             ParserDefinition::Reference(_) => {}
-            ParserDefinition::Repeat { expression, .. } => {
-                let location = location.field("repeat");
-                expression.receive(visitor, language, location.field("expression"), reporter);
-            }
             ParserDefinition::SeparatedBy { expression, .. } => {
                 let location = location.field("separatedBy");
                 expression.receive(visitor, language, location.field("expression"), reporter);

diff --git a/crates/codegen/syntax/src/first_set.rs → crates/codegen/syntax/src/char_first_set.rs b/crates/codegen/syntax/src/first_set.rs → crates/codegen/syntax/src/char_first_set.rs
@@ -1,11 +1,11 @@
 use crate::char_set::CharSet;
 
-pub struct FirstSet {
+pub struct CharFirstSet {
     pub includes_epsilon: bool,
     pub char_set: CharSet,
 }
 
-impl FirstSet {
+impl CharFirstSet {
     pub fn new() -> Self {
         Self {
             includes_epsilon: false,