From 6d8fb5e9d8145442077d5703e259091d12e57d71 Mon Sep 17 00:00:00 2001 From: John Lapeyre Date: Sat, 13 Jan 2024 02:11:36 -0500 Subject: [PATCH] Reorganize `sourcegen` from crate to submodule in `oq3_syntax` (#27) * Reorganize sourcegen from crate to submodule in oq3_syntax The sourcegen crate was published by rust-analyzer. But it was broken, I pointed this out and they said publishing at all was a mistake and removed it. So we copied the crate into our workspace. But in fact it is one file, used in one crate (oq3_syntax). So we move the code to a module (a file) in oq3_syntax. --- .gitignore | 2 +- Cargo.toml | 1 - crates/oq3_sourcegen/Cargo.toml | 18 ------------ crates/oq3_syntax/Cargo.toml | 2 +- crates/oq3_syntax/src/lib.rs | 1 + .../lib.rs => oq3_syntax/src/sourcegen.rs} | 15 ++++++---- crates/oq3_syntax/src/tests/sourcegen_ast.rs | 28 ++++++++----------- 7 files changed, 24 insertions(+), 43 deletions(-) delete mode 100644 crates/oq3_sourcegen/Cargo.toml rename crates/{oq3_sourcegen/src/lib.rs => oq3_syntax/src/sourcegen.rs} (97%) diff --git a/.gitignore b/.gitignore index 3c4b1cc..fd580a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ target/ /Cargo.lock -crates/parser/src/syntax_kind/_syntax_kind_enum.rs +crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs crates/oq3_syntax/src/ast/generated/_new_nodes.rs crates/oq3_syntax/src/ast/generated/_new_tokens.rs diff --git a/Cargo.toml b/Cargo.toml index 2ea0ae1..69fe991 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,5 +19,4 @@ oq3_lexer = { path = "crates/oq3_lexer", version = "0.0.1" } oq3_parser = { path = "crates/oq3_parser", version = "0.0.1" } oq3_syntax = { path = "crates/oq3_syntax", version = "0.0.1" } oq3_semantics = { path = "crates/oq3_semantics", version = "0.0.1" } -oq3_sourcegen = { path = "crates/oq3_sourcegen", version = "0.0.1" } oq3_source_file = { path = "crates/oq3_source_file", version = "0.0.1" } diff --git a/crates/oq3_sourcegen/Cargo.toml b/crates/oq3_sourcegen/Cargo.toml deleted file mode 100644 index 877a360..0000000 --- a/crates/oq3_sourcegen/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "oq3_sourcegen" -description = "TBD" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -authors.workspace = true -readme.workspace = true -keywords.workspace = true -categories.workspace = true -repository.workspace = true - -[lib] -doctest = false - -[dependencies] -xshell = "0.2.2" diff --git a/crates/oq3_syntax/Cargo.toml b/crates/oq3_syntax/Cargo.toml index d2cb93a..ded6a2f 100644 --- a/crates/oq3_syntax/Cargo.toml +++ b/crates/oq3_syntax/Cargo.toml @@ -26,10 +26,10 @@ rustc-hash = "1.1.0" smol_str = "0.2.0" stdx = { version = "0.0.188", package = "ra_ap_stdx"} triomphe = { version = "0.1.8", default-features = false, features = ["std"] } +xshell = "0.2.2" # local crates oq3_lexer.workspace = true oq3_parser.workspace = true -oq3_sourcegen.workspace = true [dev-dependencies] rayon = "1.6.1" diff --git a/crates/oq3_syntax/src/lib.rs b/crates/oq3_syntax/src/lib.rs index 88814ed..319aeea 100644 --- a/crates/oq3_syntax/src/lib.rs +++ b/crates/oq3_syntax/src/lib.rs @@ -35,6 +35,7 @@ macro_rules! eprintln { mod parsing; mod ptr; +mod sourcegen; mod syntax_error; pub mod syntax_node; #[cfg(test)] diff --git a/crates/oq3_sourcegen/src/lib.rs b/crates/oq3_syntax/src/sourcegen.rs similarity index 97% rename from crates/oq3_sourcegen/src/lib.rs rename to crates/oq3_syntax/src/sourcegen.rs index 1916bc8..52d4a33 100644 --- a/crates/oq3_sourcegen/src/lib.rs +++ b/crates/oq3_syntax/src/sourcegen.rs @@ -14,12 +14,6 @@ //! //! This crate contains utilities to make this kind of source-gen easy. -#![warn( - rust_2018_idioms, - unused_lifetimes, - semicolon_in_expressions_from_macros -)] - use std::{ fmt, fs, mem, path::{Path, PathBuf}, @@ -30,6 +24,7 @@ use xshell::{cmd, Shell}; // I think list_rust_files and list_files are only used to support // extracting tests from comments. +#[allow(unused)] pub fn list_rust_files(dir: &Path) -> Vec { let mut res = list_files(dir); res.retain(|it| { @@ -42,6 +37,7 @@ pub fn list_rust_files(dir: &Path) -> Vec { res } +#[allow(unused)] pub fn list_files(dir: &Path) -> Vec { let mut res = Vec::new(); let mut work = vec![dir.to_path_buf()]; @@ -77,6 +73,7 @@ pub struct CommentBlock { } impl CommentBlock { + #[allow(unused)] pub fn extract(tag: &str, text: &str) -> Vec { assert!(tag.starts_with(char::is_uppercase)); @@ -98,6 +95,7 @@ impl CommentBlock { blocks } + #[allow(unused)] pub fn extract_untagged(text: &str) -> Vec { let mut res = Vec::new(); @@ -169,6 +167,7 @@ impl fmt::Display for Location { } } +#[allow(unused)] fn ensure_rustfmt(sh: &Shell) { let version = cmd!(sh, "rustup run stable rustfmt --version") .read() @@ -181,6 +180,7 @@ fn ensure_rustfmt(sh: &Shell) { } } +#[allow(unused)] pub fn reformat(text: String) -> String { let sh = Shell::new().unwrap(); ensure_rustfmt(&sh); @@ -198,6 +198,7 @@ pub fn reformat(text: String) -> String { stdout } +#[allow(unused)] pub fn add_preamble(generator: &'static str, mut text: String) -> String { let preamble = format!("//! Generated by `{generator}`, do not edit by hand.\n\n"); text.insert_str(0, &preamble); @@ -206,6 +207,7 @@ pub fn add_preamble(generator: &'static str, mut text: String) -> String { /// Checks that the `file` has the specified `contents`. If that is not the /// case, updates the file and then fails the test. +#[allow(unused)] pub fn ensure_file_contents(file: &Path, contents: &str) { if let Ok(old_contents) = fs::read_to_string(file) { if normalize_newlines(&old_contents) == normalize_newlines(contents) { @@ -229,6 +231,7 @@ pub fn ensure_file_contents(file: &Path, contents: &str) { panic!("some file was not up to date and has been updated, simply re-run the tests"); } +#[allow(unused)] fn normalize_newlines(s: &str) -> String { s.replace("\r\n", "\n") } diff --git a/crates/oq3_syntax/src/tests/sourcegen_ast.rs b/crates/oq3_syntax/src/tests/sourcegen_ast.rs index cfb8e43..5170e62 100644 --- a/crates/oq3_syntax/src/tests/sourcegen_ast.rs +++ b/crates/oq3_syntax/src/tests/sourcegen_ast.rs @@ -14,18 +14,14 @@ use std::{ use itertools::Itertools; use proc_macro2::{Punct, Spacing}; use quote::{format_ident, quote}; +use std::path::PathBuf; use ungrammar::{Grammar, Rule}; +use crate::sourcegen as localsourcegen; use crate::tests::ast_src::{ AstEnumSrc, AstNodeSrc, AstSrc, Cardinality, Field, KindsSrc, KINDS_SRC, }; -use std::path::PathBuf; - -fn our_project_root() -> PathBuf { - oq3_sourcegen::project_root() -} - // I split this into two (now three) tests because I find tests fail if I do the second and third // codegen tasks at the same time. (GJL August 2023) @@ -36,9 +32,9 @@ fn our_project_root() -> PathBuf { #[test] fn write_syntax_kinds_enum() { let syntax_kinds = generate_syntax_kinds(KINDS_SRC); - let syntax_kinds_file = - our_project_root().join("crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs"); - oq3_sourcegen::ensure_file_contents(syntax_kinds_file.as_path(), &syntax_kinds); + let syntax_kinds_file = localsourcegen::project_root() + .join("crates/oq3_parser/src/syntax_kind/_syntax_kind_enum.rs"); + localsourcegen::ensure_file_contents(syntax_kinds_file.as_path(), &syntax_kinds); } /// Read the ungrammar from openqasm3.ungram, lower to the AST, and return the result. @@ -59,8 +55,8 @@ fn sourcegen_ast_tokens() { let ast_tokens = generate_tokens(&ast); let ast_tokens_file = - our_project_root().join("crates/oq3_syntax/src/ast/generated/_new_tokens.rs"); - oq3_sourcegen::ensure_file_contents(ast_tokens_file.as_path(), &ast_tokens); + localsourcegen::project_root().join("crates/oq3_syntax/src/ast/generated/_new_tokens.rs"); + localsourcegen::ensure_file_contents(ast_tokens_file.as_path(), &ast_tokens); } /// Generate the code destined for nodes.rs, but write to temp file _new_nodes.rs. @@ -72,8 +68,8 @@ fn sourcegen_ast_nodes() { let ast_nodes = generate_nodes(KINDS_SRC, &ast); let ast_nodes_file = - our_project_root().join("crates/oq3_syntax/src/ast/generated/_new_nodes.rs"); - oq3_sourcegen::ensure_file_contents(ast_nodes_file.as_path(), &ast_nodes); + localsourcegen::project_root().join("crates/oq3_syntax/src/ast/generated/_new_nodes.rs"); + localsourcegen::ensure_file_contents(ast_nodes_file.as_path(), &ast_nodes); } fn generate_tokens(grammar: &AstSrc) -> String { @@ -100,7 +96,7 @@ fn generate_tokens(grammar: &AstSrc) -> String { } }); - oq3_sourcegen::add_preamble( + localsourcegen::add_preamble( "sourcegen_ast", quote! { use crate::{SyntaxKind::{self, *}, SyntaxToken, ast::AstToken}; @@ -375,7 +371,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: &AstSrc) -> String { } // let res = sourcegen::add_preamble("sourcegen_ast", sourcegen::reformat(res)); FIXME - let res = oq3_sourcegen::add_preamble("sourcegen_ast", res); + let res = localsourcegen::add_preamble("sourcegen_ast", res); res.replace("#[derive", "\n#[derive") } @@ -542,7 +538,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { }; // sourcegen::add_preamble("sourcegen_ast", sourcegen::reformat(ast.to_string())) // FIXME - oq3_sourcegen::add_preamble("sourcegen_ast", ast.to_string()) + localsourcegen::add_preamble("sourcegen_ast", ast.to_string()) } fn to_upper_snake_case(s: &str) -> String {