allow sharing code between rust and napi (#543)

.changeset/gold-knives-jump.md

@@ -0,0 +1,5 @@
+---
+"changelog": patch
+---
+
+Move `syntax::parser::ProductionKind` to `syntax::nodes` namespace.

crates/codegen/legacy_syntax_templates/src/lib.rs

@@ -3,3 +3,9 @@ extern crate napi_derive;
 
 pub mod napi;
 pub mod rust;
+
+pub mod syntax {
+ pub mod nodes {
+ pub use crate::napi::kinds::{ProductionKind, RuleKind, TokenKind};
+ }
+}

crates/codegen/legacy_syntax_templates/src/napi/cst_ts_wrappers.rs

@@ -1,7 +1,6 @@
 use super::{
  cst::{Node as RustNode, RuleNode as RustRuleNode, TokenNode as RustTokenNode},
  cursor_ts_wrappers::Cursor,
- kinds::*,
  text_index::{TextIndex as RustTextIndex, TextRange as RustTextRange},
 };
 
@@ -11,6 +10,8 @@ use napi::bindgen_prelude::*;
 use napi::JsObject;
 use napi::NapiValue;
 
+use crate::syntax::nodes::{RuleKind, TokenKind};
+
 #[napi(object, namespace = "legacy")]
 #[derive(Copy, Clone)]
 pub struct TextIndex {

crates/codegen/legacy_syntax_templates/src/napi/cursor_ts_wrappers.rs

@@ -1,12 +1,13 @@
 use super::{
  cst_ts_wrappers::{TextIndex, TextRange, ToJS},
  cursor::Cursor as RustCursor,
- kinds::*,
 };
 
 use napi::bindgen_prelude::*;
 use napi::JsObject;
 
+use crate::syntax::nodes::{RuleKind, TokenKind};
+
 #[napi(namespace = "legacy")]
 pub struct Cursor(Box<RustCursor>);
 

crates/codegen/legacy_syntax_templates/src/napi/parse_output.rs

@@ -1,13 +1,15 @@
 use std::collections::BTreeSet;
 
+use napi::bindgen_prelude::*;
+
 use super::{
  cst,
  cst_ts_wrappers::{TextRange, ToJS},
- kinds::TokenKind,
  parse_error::render_error_report,
  text_index::TextRange as RustTextRange,
 };
-use napi::bindgen_prelude::*;
+
+use crate::syntax::nodes::TokenKind;
 
 #[napi(namespace = "legacy")]
 pub struct ParseOutput {

crates/codegen/legacy_syntax_templates/src/rust/parse_output.rs

@@ -1,6 +1,8 @@
 use std::collections::BTreeSet;
 
-use super::{cst, kinds::TokenKind, parse_error::render_error_report, text_index::TextRange};
+use super::{cst, parse_error::render_error_report, text_index::TextRange};
+
+use crate::syntax::nodes::TokenKind;
 
 #[derive(Debug, PartialEq)]
 pub struct ParseOutput {

crates/codegen/legacy_syntax_templates/src/shared/cst.rs

@@ -2,11 +2,9 @@ use std::rc::Rc;
 
 use serde::Serialize;
 
-use super::{
- cursor::Cursor,
- kinds::{RuleKind, TokenKind},
- text_index::TextIndex,
-};
+use super::{cursor::Cursor, text_index::TextIndex};
+
+use crate::syntax::nodes::{RuleKind, TokenKind};
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize)]
 pub struct RuleNode {

crates/codegen/legacy_syntax_templates/src/shared/cursor.rs

@@ -2,10 +2,11 @@ use std::rc::Rc;
 
 use super::{
  cst::{Node, RuleNode, TokenNode},
- kinds::{RuleKind, TokenKind},
  text_index::{TextIndex, TextRange},
 };
 
+use crate::syntax::nodes::{RuleKind, TokenKind};
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 struct CursorPathElement {
  rule_node: Rc<RuleNode>,

crates/codegen/legacy_syntax_templates/src/shared/language.rs

@@ -2,4 +2,7 @@
 use semver::Version;
 
 #[allow(unused_imports)]
-use super::{kinds::*, parse_output::*, parser_function::*, scanner_function::*};
+use super::{parse_output::*, parser_function::*, scanner_function::*};
+
+#[allow(unused_imports)]
+use crate::syntax::nodes::{ProductionKind, RuleKind, TokenKind};

crates/codegen/legacy_syntax_templates/src/shared/parse_error.rs

@@ -1,9 +1,10 @@
 use super::{
- kinds::*,
  parse_output::ParseError,
  text_index::{TextIndex, TextRange},
 };
 
+use crate::syntax::nodes::TokenKind;
+
 impl ParseError {
  #[allow(dead_code)]
  pub(crate) fn new_at_position(

crates/codegen/legacy_syntax_templates/src/shared/parser_function.rs

@@ -1,11 +1,12 @@
 use super::{
  cst,
- kinds::*,
  parse_output::{ParseError, ParseOutput},
  parser_result::*,
  stream::Stream,
 };
 
+use crate::syntax::nodes::TokenKind;
+
 // Return type of the function has to be a type parameter of the trait
 pub trait ParserFunction<L, R>
 where

crates/codegen/legacy_syntax_templates/src/shared/parser_result.rs

@@ -1,4 +1,6 @@
-use super::{cst, kinds::*, stream::Stream};
+use super::{cst, stream::Stream};
+
+use crate::syntax::nodes::{RuleKind, TokenKind};
 
 #[derive(PartialEq, Eq, Clone, Debug)]
 pub struct Match {

crates/codegen/legacy_syntax_templates/src/shared/scanner_function.rs

@@ -1,10 +1,11 @@
 use super::{
  cst,
- kinds::*,
  parse_output::{ParseError, ParseOutput},
  stream::Stream,
 };
 
+use crate::syntax::nodes::TokenKind;
+
 // Return type of the function has to be a type parameter of the trait
 pub trait ScannerFunction<L, R>
 where

crates/codegen/syntax/Cargo.toml

@@ -6,6 +6,7 @@ edition.workspace = true
 publish = false
 
 [dependencies]
+anyhow = { workspace = true }
 codegen_ebnf = { workspace = true }
 codegen_schema = { workspace = true }
 codegen_utils = { workspace = true }

crates/codegen/syntax/src/legacy/code_generator.rs

@@ -148,10 +148,7 @@ pub struct CodeGenerator {
  pub language: LanguageDefinitionRef,
  pub first_version: Version,
 
- pub token_kinds: BTreeMap<String, Option<String>>,
  pub scanners: BTreeMap<String, VersionedFunctionBody>,
-
- pub rule_kinds: BTreeSet<String>,
  pub parsers: BTreeMap<String, VersionedFunctionBody>,
 
  pub errors: Vec<String>,
@@ -163,23 +160,13 @@ impl CodeGenerator {
  language: language.clone(),
  first_version: language.versions.first().unwrap().clone(),
 
- token_kinds: Default::default(),
  scanners: Default::default(),
-
- rule_kinds: Default::default(),
  parsers: Default::default(),
 
  errors: Default::default(),
  }
  }
 
- pub fn add_token_kind(&mut self, name: String) -> Ident {
- let name = name;
- let ident = format_ident!("{name}");
- self.token_kinds.insert(name, None);
- ident
- }
-
  pub fn add_scanner(
  &mut self,
  name: String,
@@ -192,13 +179,6 @@ impl CodeGenerator {
  .insert(version, definition);
  }
 
- pub fn add_rule_kind(&mut self, name: String) -> Ident {
- let name = name;
- let ident = format_ident!("{name}");
- self.rule_kinds.insert(name);
- ident
- }
-
  pub fn add_parser(
  &mut self,
  name: String,
@@ -384,46 +364,6 @@ impl CodeGenerator {
  quote! { #(#invocations),* }
  }
 
- pub fn token_kinds(&self) -> TokenStream {
- let kinds = self
- .token_kinds
- .iter()
- .map(|(name, _)| format_ident!("{name}"));
- quote! {
- pub enum TokenKind {
- SKIPPED,
- #(#kinds),*
- }
- }
- }
-
- pub fn rule_kinds(&self) -> TokenStream {
- let kinds = self.rule_kinds.iter().map(|name| format_ident!("{name}"));
- quote! {
- pub enum RuleKind {
- #(#kinds),*
- }
- }
- }
-
- pub fn production_kinds(&self) -> TokenStream {
- let mut kinds: Vec<_> = self
- .language
- .productions
- .values()
- .filter(|production| !production.inlined)
- .map(|production| format_ident!("{}", production.name))
- .collect();
-
- kinds.sort();
-
- quote! {
- pub enum ProductionKind {
- #(#kinds),*
- }
- }
- }
-
  pub fn write_common_sources(&self, codegen: &mut CodegenContext, output_dir: &PathBuf) {
  // Rebuild if input files are added/removed
  codegen.track_input_dir(
@@ -556,11 +496,12 @@ impl CodeGenerator {
  let content = format!(
  "
  use super::cst;
- use super::kinds::*;
  use super::language::Language;
  use super::parser_helpers::*;
  use super::parser_result::*;
  use super::stream::*;
+ 
+ use crate::syntax::nodes::{{RuleKind, TokenKind}};
 
  impl Language {{
  

crates/codegen/syntax/src/legacy/combinator_tree.rs

@@ -60,7 +60,6 @@ impl<'context> CombinatorTree<'context> {
  }
 
  let name = &self.production.name;
- let inlined = &self.production.inlined;
  let comment = self.generate_comment();
 
  match self.production.definition {
@@ -72,30 +71,18 @@ impl<'context> CombinatorTree<'context> {
  );
  }
 
- if !inlined {
- code.add_token_kind(name.clone());
- }
-
  (comment, node.to_scanner_code(code))
  });
  code.add_scanner(name.clone(), version, definition);
  }
  ProductionDefinition::TriviaParser { .. } => {
- if !inlined {
- code.add_rule_kind(name.clone());
- }
-
  let definition = self
  .root_node
  .get()
  .map(|node| (comment, node.to_parser_code(code, true)));
  code.add_parser(name.clone(), version, definition);
  }
  ProductionDefinition::Parser { .. } | ProductionDefinition::PrecedenceParser { .. } => {
- if !inlined {
- code.add_rule_kind(name.clone());
- }
-
  let definition = self
  .root_node
  .get()

crates/codegen/syntax/src/legacy/napi_lib_code_generator.rs

@@ -32,7 +32,6 @@ impl CodeGenerator {
  pub mod cst_ts_wrappers;
  pub mod cursor;
  pub mod cursor_ts_wrappers;
- pub mod kinds;
  pub mod language;
  pub mod parse_error;
  pub mod parse_output;
@@ -162,59 +161,5 @@ impl CodeGenerator {
  .write_file(&output_dir.join("language.rs"), &content)
  .unwrap();
  }
-
- // Do the kinds last, because the code generation steps above may have added new kinds
- {
- let content = {
- let token_kinds = self.token_kinds();
- let rule_kinds = self.rule_kinds();
- let production_kinds = self.production_kinds();
- quote! {
- use serde::Serialize;
- use napi::bindgen_prelude::*;
- use napi_derive::napi;
-
- #[napi(string_enum, namespace = "legacy")]
- #[derive(
- Debug,
- PartialEq,
- Eq,
- Serialize,
- strum_macros::EnumString,
- strum_macros::AsRefStr,
- strum_macros::Display,
- )]
- #token_kinds
-
- #[napi(string_enum, namespace = "legacy")]
- #[derive(
- Debug,
- PartialEq,
- Eq,
- Serialize,
- strum_macros::EnumString,
- strum_macros::AsRefStr,
- strum_macros::Display,
- )]
- #rule_kinds
-
- #[napi(string_enum, namespace = "legacy")]
- #[derive(
- Debug,
- PartialEq,
- Eq,
- Serialize,
- strum_macros::EnumString,
- strum_macros::AsRefStr,
- strum_macros::Display,
- )]
- #production_kinds
- }
- };
-
- codegen
- .write_file(&output_dir.join("kinds.rs"), &content.to_string())
- .unwrap();
- }
  }
 }