From c86196dc061a2bea10363c9ed3fb7091a70e3984 Mon Sep 17 00:00:00 2001 From: Jay Oster Date: Sun, 9 Apr 2023 21:12:21 -0700 Subject: [PATCH] Initial commit --- .github/FUNDING.yml | 3 + .github/workflows/ci.yml | 79 +++++++++++++++++ .gitignore | 2 + Cargo.toml | 12 +++ LICENSE | 18 ++++ MSRV.md | 11 +++ README.md | 21 +++++ benchmarks.md | 1 + src/lib.rs | 32 +++++++ src/prelude.rs | 5 ++ src/traits.rs | 184 +++++++++++++++++++++++++++++++++++++++ src/ty.rs | 173 ++++++++++++++++++++++++++++++++++++ src/utils.rs | 43 +++++++++ 13 files changed, 584 insertions(+) create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 LICENSE create mode 100644 MSRV.md create mode 100644 README.md create mode 100644 benchmarks.md create mode 100644 src/lib.rs create mode 100644 src/prelude.rs create mode 100644 src/traits.rs create mode 100644 src/ty.rs create mode 100644 src/utils.rs diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..fc5d86b --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +github: +- parasyte +patreon: blipjoy diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fa10c0a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,79 @@ +name: CI +on: + push: + pull_request: + schedule: + - cron: '0 0 * * 0' +jobs: + checks: + name: Check + runs-on: ubuntu-latest + strategy: + matrix: + rust: + - stable + - beta + - 1.58.0 + steps: + - name: Checkout sources + uses: actions/checkout@v3 + - name: Install toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: common + - name: Cargo check + run: cargo check --workspace + + lints: + name: Lints + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v3 + - name: Install toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: common + - name: Install cargo-machete + run: cargo install --locked cargo-machete + - name: Cargo fmt + run: cargo fmt --all -- --check + - name: Cargo doc + run: cargo doc --workspace --no-deps + - name: Cargo clippy + run: cargo clippy --workspace --tests -- -D warnings + - name: Cargo machete + run: cargo machete + + tests: + name: Test + runs-on: ubuntu-latest + needs: [checks, lints] + strategy: + matrix: + rust: + - stable + - beta + - 1.58.0 + steps: + - name: Checkout sources + uses: actions/checkout@v3 + - name: Install toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: common + - name: Cargo test + run: cargo test --workspace diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4fffb2f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +/Cargo.lock diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..31ec562 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "myn" +description = "Minimalist Rust syntax parsing for procedural macros" +version = "0.1.0" +authors = ["Jay Oster "] +edition = "2021" +keywords = ["macros", "myn"] +categories = ["parser-implementations", "procedural-macro-helpers"] +license = "MIT" + +[dependencies] +# No dependencies! diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d4ae05b --- /dev/null +++ b/LICENSE @@ -0,0 +1,18 @@ +Copyright 2023 Jay Oster + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/MSRV.md b/MSRV.md new file mode 100644 index 0000000..a90d2ea --- /dev/null +++ b/MSRV.md @@ -0,0 +1,11 @@ +# Minimum Supported Rust Version + +| `myn` version | `rustc` version | +|---------------|-----------------| +| (unreleased) | `1.58.0` | + +## Policy + +The table above will be kept up-to-date in lock-step with CI on the main branch in GitHub. It may contain information about unreleased and yanked versions. It is the user's responsibility to consult with the [`myn` versions page](https://crates.io/crates/myn/versions) on `crates.io` to verify version status. + +The MSRV will be chosen as the minimum version of `rustc` that can successfully pass CI, including documentation, lints, and all examples. For this reason, the minimum version _supported_ may be higher than the minimum version _required_ to compile the `myn` crate itself. See `Cargo.toml` for the minimal Rust version required to build the crate alone. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ef28dac --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +[![Crates.io](https://img.shields.io/crates/v/myn)](https://crates.io/crates/myn "Crates.io version") +[![Documentation](https://img.shields.io/docsrs/myn)](https://docs.rs/myn "Documentation") +[![unsafe forbidden](https://img.shields.io/badge/unsafe-forbidden-success.svg)](https://github.com/rust-secure-code/safety-dance/) +[![GitHub actions](https://img.shields.io/github/actions/workflow/status/parasyte/myn/ci.yml?branch=main)](https://github.com/parasyte/myn/actions "CI") +[![GitHub activity](https://img.shields.io/github/last-commit/parasyte/myn)](https://github.com/parasyte/myn/commits "Commit activity") +[![GitHub Sponsors](https://img.shields.io/github/sponsors/parasyte)](https://github.com/sponsors/parasyte "Sponsors") + +Minimalist Rust syntax parsing for procedural macros. + +You can think of `myn` as a minimalist crate with similarities to [`syn`](https://docs.rs/syn). It provides utilities to help write procedural macros, but does not attempt to replicate the `syn` types or API. + +`myn` exists to support a very small subset of the entire Rust language syntax. Just enough to implement `#[derive]` macros on `struct`s and `enum`s, and that's about it. Everything else is currently out of scope. + +## Why + +- 100% safe Rust 🦀. +- Write `#[derive]` macros with extremely fast compile times. See [benchmarks](./benchmark.md). + +## MSRV Policy + +The Minimum Supported Rust Version for `myn` will always be made available in the [MSRV.md](./MSRV.md) file on GitHub. diff --git a/benchmarks.md b/benchmarks.md new file mode 100644 index 0000000..6a348f7 --- /dev/null +++ b/benchmarks.md @@ -0,0 +1 @@ +# TDB diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..3a36f3b --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,32 @@ +//! Minimalist Rust syntax parsing for procedural macros. +//! +//! # Rationale +//! +//! You may wonder why this is even a thing, since `syn` already exists, is a well-supported and +//! excellent crate, and it supports the entire gamut of Rust syntax. In short, `syn` hurts compile +//! times and is almost certainly overkill for your use case. +//! +//! Instead, we prefer a "pay for what you use" model. This small surface area affords rapid compile +//! times at the cost of being able to parse the entirety of Rust language syntax. This is right +//! tradeoff for `#[derive]` macros where compile time is of high importance. +//! +//! # Where to begin +//! +//! `myn` works directly with [`TokenStream`], giving you tools to build your own AST without +//! attempting to define a one-size-fits-all strongly typed AST. The [`TokenStreamExt`] extension +//! trait turns the `TokenStream` into a [`TokenIter`]. +//! +//! [`TokenIter`]: crate::ty::TokenIter +//! [`TokenStream`]: proc_macro::TokenStream +//! [`TokenStreamExt`]: crate::traits::TokenStreamExt + +#![forbid(unsafe_code)] +#![deny(clippy::all)] +#![deny(clippy::pedantic)] + +extern crate proc_macro; + +pub mod prelude; +pub mod traits; +pub mod ty; +pub mod utils; diff --git a/src/prelude.rs b/src/prelude.rs new file mode 100644 index 0000000..4f36d77 --- /dev/null +++ b/src/prelude.rs @@ -0,0 +1,5 @@ +//! Re-exports all public items. + +pub use crate::traits::*; +pub use crate::ty::*; +pub use crate::utils::*; diff --git a/src/traits.rs b/src/traits.rs new file mode 100644 index 0000000..90e6e7a --- /dev/null +++ b/src/traits.rs @@ -0,0 +1,184 @@ +//! Extension traits. +//! +//! The primary trait is [`TokenIterExt`], which provides the parsers. + +use crate::ty::{Attribute, TokenIter}; +use crate::utils::spanned_error; +use proc_macro::{Delimiter, Group, Ident, Literal, Punct, Span, TokenStream, TokenTree}; + +/// An extension trait for [`TokenStream`]. +pub trait TokenStreamExt { + /// Turn this type into a [`TokenIter`]. + fn into_token_iter(self) -> TokenIter; +} + +/// An extension trait for [`TokenIter`]. +/// +/// This trait provides parsers and shorthand methods for common getter patterns. +pub trait TokenIterExt: Iterator { + /// Parse the input iterator into a list of attributes. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn parse_attributes(&mut self) -> Result, TokenStream>; + + /// Parse the input iterator as a type visibility modifier. + /// + /// E.g. `pub` or `pub(super)`. + /// + /// This parser currently discards the result, since it doesn't have much use in a derive macro. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn parse_visibility(&mut self) -> Result<(), TokenStream>; + + /// Parse the input iterator as a path into a string/span pair. + /// + /// E.g. `std::collections::HashMap`. + /// + /// Due to current limitations in the [`Span`] API, the returned span only points at the span + /// for the first path segment. For example, it would be `std` in the path above. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn parse_path(&mut self) -> Result<(String, Span), TokenStream>; + + /// Parse the input as a group, expecting the given delimiter. + /// + /// Returns the group's inner [`TokenStream`] as a [`TokenIter`] when successful. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn expect_group(&mut self, expect: Delimiter) -> Result; + + /// Parse the input as an identifier, expecting it to match the given string. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn expect_ident(&mut self, expect: &str) -> Result<(), TokenStream>; + + /// Parse the input as punctuation, expecting it to match the given char. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn expect_punct(&mut self, expect: char) -> Result<(), TokenStream>; + + /// Parse the input as a group. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn as_group(&mut self) -> Result; + + /// Parse the input as an identifier. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn as_ident(&mut self) -> Result; + + /// Parse the input as a literal. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn as_lit(&mut self) -> Result; + + /// Parse the input as punctuation. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn as_punct(&mut self) -> Result; +} + +/// An extension trait for [`TokenTree`]. +pub trait TokenTreeExt { + /// Get a span from the given [`TokenTree`]. + fn as_span(&self) -> Span; +} + +/// An extension trait for [`Literal`]. +pub trait LiteralExt { + /// Parse a literal into a char. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn as_char(&self) -> Result; + + /// Parse a literal into a string. + /// + /// # Errors + /// + /// Returns a compiler error if parsing fails. The error should be inserted into the + /// `proc_macro` stream. + fn as_string(&self) -> Result; +} + +impl TokenStreamExt for TokenStream { + fn into_token_iter(self) -> TokenIter { + self.into_iter().peekable() + } +} + +impl TokenTreeExt for Option { + fn as_span(&self) -> Span { + match self { + Some(TokenTree::Group(group)) => group.span(), + Some(TokenTree::Ident(ident)) => ident.span(), + Some(TokenTree::Punct(punct)) => punct.span(), + Some(TokenTree::Literal(lit)) => lit.span(), + None => Span::call_site(), + } + } +} + +impl LiteralExt for Literal { + fn as_char(&self) -> Result { + let string = format!("{self}"); + if !string.starts_with('\'') || !string.ends_with('\'') { + return Err(spanned_error("Expected char literal", self.span())); + } + + // Strip single quotes. + string + .chars() + .nth(1) + .ok_or_else(|| spanned_error("Expected char literal", self.span())) + } + + fn as_string(&self) -> Result { + let string = format!("{self}"); + if !string.starts_with('"') || !string.ends_with('"') { + return Err(spanned_error("Expected string literal", self.span())); + } + + // Strip double quotes and escapes. + Ok(string[1..string.len() - 1] + .trim() + .replace(r#"\""#, r#"""#) + .replace(r"\n", "\n") + .replace(r"\r", "\r") + .replace(r"\t", "\t") + .replace(r"\'", "'") + .replace(r"\\", r"\")) + } +} diff --git a/src/ty.rs b/src/ty.rs new file mode 100644 index 0000000..d4e6947 --- /dev/null +++ b/src/ty.rs @@ -0,0 +1,173 @@ +//! High-level types from the parser. + +use crate::traits::{TokenIterExt, TokenStreamExt as _, TokenTreeExt as _}; +use crate::utils::spanned_error; +use proc_macro::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::iter::Peekable; + +/// A type alias for the primary [`TokenTree`] iterator. +/// +/// It is [`Peekable`] to allow look-ahead of single items. +pub type TokenIter = Peekable; + +/// A type representing `#[attributes]`. +pub struct Attribute { + /// The attribute name. + /// + /// This would be `hello` for `#[hello]`. + pub name: Ident, + + /// The inner [`TokenTree`] iterator. + pub tree: TokenIter, +} + +impl TokenIterExt for TokenIter { + fn parse_attributes(&mut self) -> Result, TokenStream> { + let mut attrs = vec![]; + + loop { + match self.peek() { + Some(TokenTree::Punct(punct)) if punct.as_char() == '#' => self.next(), + _ => break, + }; + + let mut group = self.expect_group(Delimiter::Bracket)?; + let ident = group.as_ident()?; + + attrs.push(Attribute { + name: ident, + tree: group.collect::().into_token_iter(), + }); + } + + Ok(attrs) + } + + fn parse_visibility(&mut self) -> Result<(), TokenStream> { + match self.peek() { + Some(TokenTree::Ident(ident)) if ident.to_string() == "pub" => self.next(), + _ => return Ok(()), + }; + + match self.peek() { + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::Parenthesis => { + self.next(); + } + _ => return Ok(()), + } + + Ok(()) + } + + fn parse_path(&mut self) -> Result<(String, Span), TokenStream> { + let mut path = String::new(); + let mut span = None; + let mut nesting = 0; + + while let Some(tree) = self.peek() { + match tree { + TokenTree::Punct(punct) if punct.as_char() == ',' && nesting == 0 => break, + TokenTree::Punct(punct) => { + let ch = punct.as_char(); + + // Handle nesting with `<...>` + if ch == '<' { + nesting += 1; + } else if ch == '>' && punct.spacing() == Spacing::Joint { + nesting -= 1; + } + + span.get_or_insert_with(|| punct.span()); + path.push(ch); + } + TokenTree::Ident(ident) => { + span.get_or_insert_with(|| ident.span()); + path.push_str(&ident.to_string()); + } + _ => return Err(spanned_error("Unexpected token", self.next().as_span())), + } + + self.next(); + } + + let span = + span.ok_or_else(|| spanned_error("Unexpected end of stream", Span::call_site()))?; + + Ok((path, span)) + } + + fn expect_group(&mut self, expect: Delimiter) -> Result { + self.as_group().and_then(|group| { + let delim = group.delimiter(); + if delim == expect { + Ok(group.stream().into_token_iter()) + } else { + let expect = match expect { + Delimiter::Brace => "{", + Delimiter::Bracket => "[", + Delimiter::None => "delimiter", + Delimiter::Parenthesis => "(", + }; + + Err(spanned_error(format!("Expected `{expect}`"), group.span())) + } + }) + } + + fn expect_ident(&mut self, expect: &str) -> Result<(), TokenStream> { + self.as_ident().and_then(|ident| { + if ident.to_string() == expect { + Ok(()) + } else { + Err(spanned_error(format!("Expected `{expect}`"), ident.span())) + } + }) + } + + fn expect_punct(&mut self, expect: char) -> Result<(), TokenStream> { + self.as_punct().and_then(|punct| { + if punct.as_char() == expect { + Ok(()) + } else { + Err(spanned_error(format!("Expected `{expect}`"), punct.span())) + } + }) + } + + fn as_group(&mut self) -> Result { + match self.next() { + Some(TokenTree::Group(group)) => Ok(group), + tree => Err(spanned_error("Expected group", tree.as_span())), + } + } + + fn as_ident(&mut self) -> Result { + match self.next() { + Some(TokenTree::Ident(ident)) => Ok(ident), + tree => Err(spanned_error("Expected identifier", tree.as_span())), + } + } + + fn as_lit(&mut self) -> Result { + match self.next() { + Some(TokenTree::Literal(lit)) => Ok(lit), + tree => Err(spanned_error("Expected literal", tree.as_span())), + } + } + + fn as_punct(&mut self) -> Result { + match self.next() { + Some(TokenTree::Punct(punct)) => Ok(punct), + tree => Err(spanned_error("Expected punctuation", tree.as_span())), + } + } +} + +impl std::fmt::Debug for Attribute { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + f.debug_struct("Attribute") + .field("name", &self.name) + .field("tree", &"TokenIter {...}") + .finish() + } +} diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..24f0279 --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,43 @@ +//! Miscellaneous functions. + +use crate::traits::{LiteralExt as _, TokenIterExt as _}; +use crate::ty::Attribute; +use proc_macro::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; + +/// Create a compiler error with the given span. +pub fn spanned_error>(msg: S, span: Span) -> TokenStream { + let mut group = Group::new( + Delimiter::Parenthesis, + TokenTree::from(Literal::string(msg.as_ref())).into(), + ); + group.set_span(span); + + TokenStream::from_iter([ + TokenTree::Ident(Ident::new("compile_error", span)), + TokenTree::Punct(Punct::new('!', Spacing::Alone)), + TokenTree::Group(group), + TokenTree::Punct(Punct::new(';', Spacing::Alone)), + ]) +} + +/// Get a list of lines representing the doc comment. +#[must_use] +pub fn get_doc_comment(attrs: &[Attribute]) -> Vec { + attrs + .iter() + .filter_map(|attr| { + if attr.name.to_string() == "doc" { + let mut tree = attr.tree.clone(); + + match tree.next() { + Some(TokenTree::Punct(punct)) if punct.as_char() == '=' => (), + _ => return None, + } + + tree.as_lit().and_then(|lit| lit.as_string()).ok() + } else { + None + } + }) + .collect() +}