Reorganize sourcegen from crate to submodule in oq3_syntax (#27)

* Reorganize sourcegen from crate to submodule in oq3_syntax

The sourcegen crate was published by rust-analyzer. But it was
broken, I pointed this out and they said publishing at all
was a mistake and removed it.

So we copied the crate into our workspace. But in fact it is
one file, used in one crate (oq3_syntax). So we move the
code to a module (a file) in oq3_syntax.

.gitignore

@@ -1,5 +1,5 @@
 target/
 /Cargo.lock
-crates/parser/src/syntax_kind/_syntax_kind_enum.rs
+crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs
 crates/oq3_syntax/src/ast/generated/_new_nodes.rs
 crates/oq3_syntax/src/ast/generated/_new_tokens.rs 

Cargo.toml

@@ -19,5 +19,4 @@ oq3_lexer = { path = "crates/oq3_lexer", version = "0.0.1" }
 oq3_parser = { path = "crates/oq3_parser", version = "0.0.1" }
 oq3_syntax = { path = "crates/oq3_syntax", version = "0.0.1" }
 oq3_semantics = { path = "crates/oq3_semantics", version = "0.0.1" }
-oq3_sourcegen = { path = "crates/oq3_sourcegen", version = "0.0.1" }
 oq3_source_file = { path = "crates/oq3_source_file", version = "0.0.1" }

crates/oq3_syntax/Cargo.toml

@@ -26,10 +26,10 @@ rustc-hash = "1.1.0"
 smol_str = "0.2.0"
 stdx = { version = "0.0.188", package = "ra_ap_stdx"}
 triomphe = { version = "0.1.8", default-features = false, features = ["std"] }
+xshell = "0.2.2"
 # local crates
 oq3_lexer.workspace = true
 oq3_parser.workspace = true
-oq3_sourcegen.workspace = true
 
 [dev-dependencies]
 rayon = "1.6.1"

crates/oq3_syntax/src/lib.rs

@@ -35,6 +35,7 @@ macro_rules! eprintln {
 
 mod parsing;
 mod ptr;
+mod sourcegen;
 mod syntax_error;
 pub mod syntax_node;
 #[cfg(test)]

crates/oq3_sourcegen/src/lib.rs → crates/oq3_syntax/src/sourcegen.rs

@@ -14,12 +14,6 @@
 //!
 //! This crate contains utilities to make this kind of source-gen easy.
 
-#![warn(
- rust_2018_idioms,
- unused_lifetimes,
- semicolon_in_expressions_from_macros
-)]
-
 use std::{
  fmt, fs, mem,
  path::{Path, PathBuf},
@@ -30,6 +24,7 @@ use xshell::{cmd, Shell};
 // I think list_rust_files and list_files are only used to support
 // extracting tests from comments.
 
+#[allow(unused)]
 pub fn list_rust_files(dir: &Path) -> Vec<PathBuf> {
  let mut res = list_files(dir);
  res.retain(|it| {
@@ -42,6 +37,7 @@ pub fn list_rust_files(dir: &Path) -> Vec<PathBuf> {
  res
 }
 
+#[allow(unused)]
 pub fn list_files(dir: &Path) -> Vec<PathBuf> {
  let mut res = Vec::new();
  let mut work = vec![dir.to_path_buf()];
@@ -77,6 +73,7 @@ pub struct CommentBlock {
 }
 
 impl CommentBlock {
+ #[allow(unused)]
  pub fn extract(tag: &str, text: &str) -> Vec<CommentBlock> {
  assert!(tag.starts_with(char::is_uppercase));
 
@@ -98,6 +95,7 @@ impl CommentBlock {
  blocks
  }
 
+ #[allow(unused)]
  pub fn extract_untagged(text: &str) -> Vec<CommentBlock> {
  let mut res = Vec::new();
 
@@ -169,6 +167,7 @@ impl fmt::Display for Location {
  }
 }
 
+#[allow(unused)]
 fn ensure_rustfmt(sh: &Shell) {
  let version = cmd!(sh, "rustup run stable rustfmt --version")
  .read()
@@ -181,6 +180,7 @@ fn ensure_rustfmt(sh: &Shell) {
  }
 }
 
+#[allow(unused)]
 pub fn reformat(text: String) -> String {
  let sh = Shell::new().unwrap();
  ensure_rustfmt(&sh);
@@ -198,6 +198,7 @@ pub fn reformat(text: String) -> String {
  stdout
 }
 
+#[allow(unused)]
 pub fn add_preamble(generator: &'static str, mut text: String) -> String {
  let preamble = format!("//! Generated by `{generator}`, do not edit by hand.\n\n");
  text.insert_str(0, &preamble);
@@ -206,6 +207,7 @@ pub fn add_preamble(generator: &'static str, mut text: String) -> String {
 
 /// Checks that the `file` has the specified `contents`. If that is not the
 /// case, updates the file and then fails the test.
+#[allow(unused)]
 pub fn ensure_file_contents(file: &Path, contents: &str) {
  if let Ok(old_contents) = fs::read_to_string(file) {
  if normalize_newlines(&old_contents) == normalize_newlines(contents) {
@@ -229,6 +231,7 @@ pub fn ensure_file_contents(file: &Path, contents: &str) {
  panic!("some file was not up to date and has been updated, simply re-run the tests");
 }
 
+#[allow(unused)]
 fn normalize_newlines(s: &str) -> String {
  s.replace("\r\n", "\n")
 }

crates/oq3_syntax/src/tests/sourcegen_ast.rs

@@ -14,18 +14,14 @@ use std::{
 use itertools::Itertools;
 use proc_macro2::{Punct, Spacing};
 use quote::{format_ident, quote};
+use std::path::PathBuf;
 use ungrammar::{Grammar, Rule};
 
+use crate::sourcegen as localsourcegen;
 use crate::tests::ast_src::{
  AstEnumSrc, AstNodeSrc, AstSrc, Cardinality, Field, KindsSrc, KINDS_SRC,
 };
 
-use std::path::PathBuf;
-
-fn our_project_root() -> PathBuf {
- oq3_sourcegen::project_root()
-}
-
 // I split this into two (now three) tests because I find tests fail if I do the second and third
 // codegen tasks at the same time. (GJL August 2023)
 
@@ -36,9 +32,9 @@ fn our_project_root() -> PathBuf {
 #[test]
 fn write_syntax_kinds_enum() {
  let syntax_kinds = generate_syntax_kinds(KINDS_SRC);
- let syntax_kinds_file =
- our_project_root().join("crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs");
- oq3_sourcegen::ensure_file_contents(syntax_kinds_file.as_path(), &syntax_kinds);
+ let syntax_kinds_file = localsourcegen::project_root()
+ .join("crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs");
+ localsourcegen::ensure_file_contents(syntax_kinds_file.as_path(), &syntax_kinds);
 }
 
 /// Read the ungrammar from openqasm3.ungram, lower to the AST, and return the result.
@@ -59,8 +55,8 @@ fn sourcegen_ast_tokens() {
 
  let ast_tokens = generate_tokens(&ast);
  let ast_tokens_file =
- our_project_root().join("crates/oq3_syntax/src/ast/generated/_new_tokens.rs");
- oq3_sourcegen::ensure_file_contents(ast_tokens_file.as_path(), &ast_tokens);
+ localsourcegen::project_root().join("crates/oq3_syntax/src/ast/generated/_new_tokens.rs");
+ localsourcegen::ensure_file_contents(ast_tokens_file.as_path(), &ast_tokens);
 }
 
 /// Generate the code destined for nodes.rs, but write to temp file _new_nodes.rs.
@@ -72,8 +68,8 @@ fn sourcegen_ast_nodes() {
 
  let ast_nodes = generate_nodes(KINDS_SRC, &ast);
  let ast_nodes_file =
- our_project_root().join("crates/oq3_syntax/src/ast/generated/_new_nodes.rs");
- oq3_sourcegen::ensure_file_contents(ast_nodes_file.as_path(), &ast_nodes);
+ localsourcegen::project_root().join("crates/oq3_syntax/src/ast/generated/_new_nodes.rs");
+ localsourcegen::ensure_file_contents(ast_nodes_file.as_path(), &ast_nodes);
 }
 
 fn generate_tokens(grammar: &AstSrc) -> String {
@@ -100,7 +96,7 @@ fn generate_tokens(grammar: &AstSrc) -> String {
  }
  });
 
- oq3_sourcegen::add_preamble(
+ localsourcegen::add_preamble(
  "sourcegen_ast",
  quote! {
  use crate::{SyntaxKind::{self, *}, SyntaxToken, ast::AstToken};
@@ -375,7 +371,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: &AstSrc) -> String {
  }
 
  // let res = sourcegen::add_preamble("sourcegen_ast", sourcegen::reformat(res)); FIXME
- let res = oq3_sourcegen::add_preamble("sourcegen_ast", res);
+ let res = localsourcegen::add_preamble("sourcegen_ast", res);
  res.replace("#[derive", "\n#[derive")
 }
 
@@ -542,7 +538,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
  };
 
  // sourcegen::add_preamble("sourcegen_ast", sourcegen::reformat(ast.to_string())) // FIXME
- oq3_sourcegen::add_preamble("sourcegen_ast", ast.to_string())
+ localsourcegen::add_preamble("sourcegen_ast", ast.to_string())
 }
 
 fn to_upper_snake_case(s: &str) -> String {