From acd676652368a52e4f4cde0bdd4046f75ba2184c Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Thu, 30 Aug 2018 08:37:55 +0200 Subject: [PATCH 1/7] run cargo fmt --- cpp_build/src/lib.rs | 135 ++++++++++++++++++++++++++++-------------- cpp_common/src/lib.rs | 20 +++---- cpp_macros/src/lib.rs | 54 +++++++++-------- 3 files changed, 128 insertions(+), 81 deletions(-) diff --git a/cpp_build/src/lib.rs b/cpp_build/src/lib.rs index 7ebbf3f..d292f7f 100644 --- a/cpp_build/src/lib.rs +++ b/cpp_build/src/lib.rs @@ -5,8 +5,8 @@ //! documentation](https://docs.rs/cpp). extern crate cpp_common; -extern crate cpp_synom as synom; extern crate cpp_syn as syn; +extern crate cpp_synom as synom; extern crate cpp_synmap; @@ -15,15 +15,17 @@ extern crate cc; #[macro_use] extern crate lazy_static; +use cpp_common::{ + flags, parsing, Capture, Class, Closure, ClosureSig, Macro, FILE_HASH, LIB_NAME, OUT_DIR, + STRUCT_METADATA_MAGIC, VERSION, +}; +use cpp_synmap::SourceMap; use std::env; -use std::path::{Path, PathBuf}; use std::fs::{create_dir, remove_dir_all, File}; use std::io::prelude::*; +use std::path::{Path, PathBuf}; use syn::visit::Visitor; -use syn::{Mac, Spanned, DUMMY_SPAN, Ident, Token, TokenTree}; -use cpp_common::{parsing, Capture, Closure, ClosureSig, Macro, Class, LIB_NAME, STRUCT_METADATA_MAGIC, - VERSION, OUT_DIR, FILE_HASH, flags}; -use cpp_synmap::SourceMap; +use syn::{Ident, Mac, Spanned, Token, TokenTree, DUMMY_SPAN}; fn warnln_impl(a: String) { for s in a.lines() { @@ -41,8 +43,8 @@ macro_rules! warnln { // Note: if the this macro is itself in a macro, it should be on on the same line of the macro macro_rules! add_line { ($e:expr) => { - concat!("#line ", line!() , " \"", file!(), "\"\n", $e) - } + concat!("#line ", line!(), " \"", file!(), "\"\n", $e) + }; } const INTERNAL_CPP_STRUCTS: &'static str = add_line!(r#" @@ -128,13 +130,13 @@ template int compare_helper(const T &a, const T&b, int cmp) { lazy_static! { static ref CPP_DIR: PathBuf = OUT_DIR.join("rust_cpp"); - - static ref CARGO_MANIFEST_DIR: PathBuf = - PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect(r#" + static ref CARGO_MANIFEST_DIR: PathBuf = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect( + r#" -- rust-cpp fatal error -- The CARGO_MANIFEST_DIR environment variable was not set. -NOTE: rust-cpp's build function must be run in a build script."#)); +NOTE: rust-cpp's build function must be run in a build script."# + )); } enum ExpandSubMacroType<'a> { @@ -160,35 +162,54 @@ fn expand_sub_rust_macro(input: String, mut t: ExpandSubMacroType) -> String { } ExpandSubMacroType::Closure(ref mut offset) => { **offset += 1; - format!("rust_cpp_callbacks{file_hash}[{offset}]", - file_hash = *FILE_HASH, offset = **offset - 1).into() + format!( + "rust_cpp_callbacks{file_hash}[{offset}]", + file_hash = *FILE_HASH, + offset = **offset - 1 + ).into() } }; - let mut decl_types = rust_invocation.arguments.iter().map(|&(_, ref val)| { - format!("rustcpp::argument_helper<{}>::type", val) - }).collect::>(); - let mut call_args = rust_invocation.arguments.iter().map(|&(ref val, _)| val.as_ref()).collect::>(); + let mut decl_types = rust_invocation + .arguments + .iter() + .map(|&(_, ref val)| format!("rustcpp::argument_helper<{}>::type", val)) + .collect::>(); + let mut call_args = rust_invocation + .arguments + .iter() + .map(|&(ref val, _)| val.as_ref()) + .collect::>(); let fn_call = match rust_invocation.return_type { - None => { - format!("reinterpret_cast({f})({args})", - f = fn_name, types = decl_types.join(", "), args = call_args.join(", ")) - } + None => format!( + "reinterpret_cast({f})({args})", + f = fn_name, + types = decl_types.join(", "), + args = call_args.join(", ") + ), Some(rty) => { decl_types.push(format!("rustcpp::return_helper<{rty}>", rty = rty)); call_args.push("0"); - format!("std::move(*reinterpret_cast<{rty}*(*)({types})>({f})({args}))", - rty = rty, f = fn_name, types = decl_types.join(", "), args = call_args.join(", ")) + format!( + "std::move(*reinterpret_cast<{rty}*(*)({types})>({f})({args}))", + rty = rty, + f = fn_name, + types = decl_types.join(", "), + args = call_args.join(", ") + ) } }; let fn_call = { // remove the rust! macro from the C++ snippet - let orig = result.drain(rust_invocation.begin .. rust_invocation.end); + let orig = result.drain(rust_invocation.begin..rust_invocation.end); // add \ņ to the invocation in order to keep the same amount of line numbers // so errors point to the right line. - orig.filter(|x| *x == '\n').fold(fn_call, |mut res, _| {res.push('\n'); res}) + orig.filter(|x| *x == '\n').fold(fn_call, |mut res, _| { + res.push('\n'); + res + }) }; // add the invocation of call where the rust! macro used to be. result.insert_str(rust_invocation.begin, &fn_call); @@ -207,18 +228,29 @@ fn gen_cpp_lib(visitor: &Handle) -> PathBuf { write!(output, "{}", INTERNAL_CPP_STRUCTS).unwrap(); if visitor.callbacks_count > 0 { - write!(output, add_line!(r#" + write!( + output, + add_line!( + r#" extern "C" {{ void (*rust_cpp_callbacks{file_hash}[{callbacks_count}])() = {{}}; }} - "#), file_hash = *FILE_HASH, callbacks_count = visitor.callbacks_count).unwrap(); + "# + ), + file_hash = *FILE_HASH, + callbacks_count = visitor.callbacks_count + ).unwrap(); } - write!(output, "{}\n\n", &visitor.snippets).unwrap(); let mut sizealign = vec![]; - for &Closure { ref body, ref sig, ref callback_offset } in &visitor.closures { + for &Closure { + ref body, + ref sig, + ref callback_offset, + } in &visitor.closures + { let &ClosureSig { ref captures, ref cpp, @@ -262,23 +294,29 @@ extern "C" {{ mutable, ref name, ref cpp, - }| if mutable { - format!("{} & {}", cpp, name) - } else { - format!("{} const& {}", cpp, name) + }| { + if mutable { + format!("{} & {}", cpp, name) + } else { + format!("{} const& {}", cpp, name) + } }, ) .collect::>() .join(", "); if is_void { - write!(output, add_line!(r#" + write!( + output, + add_line!( + r#" extern "C" {{ void {name}({params}) {{ {body} }} }} -"#), +"# + ), name = &name, params = params, body = body.node @@ -290,7 +328,10 @@ void {name}({params}) {{ .map(|&Capture { ref name, .. }| name.as_ref()) .collect::>() .join(", "); - write!(output, add_line!(r#" + write!( + output, + add_line!( + r#" static inline {ty} {name}_impl({params}) {{ {body} }} @@ -299,7 +340,8 @@ void {name}({params}{comma} void* __result) {{ ::new(__result) ({ty})({name}_impl({args})); }} }} -"#), +"# + ), name = &name, params = params, comma = comma, @@ -454,7 +496,10 @@ impl Config { pub fn new() -> Config { let mut cc = cc::Build::new(); cc.cpp(true).include(&*CARGO_MANIFEST_DIR); - Config { cc: cc, std_flag_set: false } + Config { + cc: cc, + std_flag_set: false, + } } /// Add a directory to the `-I` or include path for headers @@ -723,12 +768,12 @@ struct Handle<'a> { } fn line_directive(span: syn::Span, sm: &SourceMap) -> String { - let loc = sm.locinfo(span).unwrap(); - let mut line = format!("#line {} {:?}\n", loc.line, loc.path); - for _ in 0..loc.col { - line.push(' '); - } - return line; + let loc = sm.locinfo(span).unwrap(); + let mut line = format!("#line {} {:?}\n", loc.line, loc.path); + for _ in 0..loc.col { + line.push(' '); + } + return line; } fn extract_with_span(spanned: &mut Spanned, src: &str, offset: usize, sm: &SourceMap) { diff --git a/cpp_common/src/lib.rs b/cpp_common/src/lib.rs index 724d392..148f462 100644 --- a/cpp_common/src/lib.rs +++ b/cpp_common/src/lib.rs @@ -11,9 +11,9 @@ extern crate quote; extern crate lazy_static; use std::collections::hash_map::DefaultHasher; +use std::env; use std::hash::{Hash, Hasher}; use std::path::PathBuf; -use std::env; use syn::{Ident, MetaItem, Spanned, Ty}; @@ -23,11 +23,11 @@ pub const LIB_NAME: &'static str = "librust_cpp_generated.a"; pub const MSVC_LIB_NAME: &'static str = "rust_cpp_generated.lib"; pub mod flags { - pub const IS_COPY_CONSTRUCTIBLE : u32 = 0; - pub const IS_DEFAULT_CONSTRUCTIBLE : u32 = 1; - pub const IS_TRIVIALLY_DESTRUCTIBLE : u32 = 2; - pub const IS_TRIVIALLY_COPYABLE : u32 = 3; - pub const IS_TRIVIALLY_DEFAULT_CONSTRUCTIBLE : u32 = 4; + pub const IS_COPY_CONSTRUCTIBLE: u32 = 0; + pub const IS_DEFAULT_CONSTRUCTIBLE: u32 = 1; + pub const IS_TRIVIALLY_DESTRUCTIBLE: u32 = 2; + pub const IS_TRIVIALLY_COPYABLE: u32 = 3; + pub const IS_TRIVIALLY_DEFAULT_CONSTRUCTIBLE: u32 = 4; } /// This constant is expected to be a unique string within the compiled binary @@ -51,13 +51,13 @@ pub const STRUCT_METADATA_MAGIC: [u8; 128] = [ ]; lazy_static! { - pub static ref OUT_DIR: PathBuf = - PathBuf::from(env::var("OUT_DIR").expect(r#" + pub static ref OUT_DIR: PathBuf = PathBuf::from(env::var("OUT_DIR").expect( + r#" -- rust-cpp fatal error -- The OUT_DIR environment variable was not set. -NOTE: rustc must be run by Cargo."#)); - +NOTE: rustc must be run by Cargo."# + )); pub static ref FILE_HASH: u64 = { let mut hasher = std::collections::hash_map::DefaultHasher::new(); OUT_DIR.hash(&mut hasher); diff --git a/cpp_macros/src/lib.rs b/cpp_macros/src/lib.rs index 2366a51..1a69733 100644 --- a/cpp_macros/src/lib.rs +++ b/cpp_macros/src/lib.rs @@ -23,37 +23,39 @@ extern crate aho_corasick; extern crate byteorder; -use std::collections::HashMap; +use cpp_common::{flags, parsing, FILE_HASH, LIB_NAME, MSVC_LIB_NAME, OUT_DIR, VERSION}; use proc_macro::TokenStream; -use cpp_common::{parsing, LIB_NAME, MSVC_LIB_NAME, VERSION, OUT_DIR, FILE_HASH, flags}; +use std::collections::HashMap; use syn::Ident; -use std::io::{self, BufReader, Read, Seek, SeekFrom}; -use std::fs::File; use aho_corasick::{AcAutomaton, Automaton}; use byteorder::{LittleEndian, ReadBytesExt}; +use std::fs::File; +use std::io::{self, BufReader, Read, Seek, SeekFrom}; struct MetaData { size: usize, align: usize, - flags: u64 + flags: u64, } impl MetaData { - fn has_flag(&self, f : u32) -> bool { + fn has_flag(&self, f: u32) -> bool { self.flags & (1 << f) != 0 } } lazy_static! { static ref METADATA: HashMap> = { - let file = open_lib_file().expect(r#" + let file = open_lib_file().expect( + r#" -- rust-cpp fatal error -- Failed to open the target library file. -NOTE: Did you make sure to add the rust-cpp build script?"#); +NOTE: Did you make sure to add the rust-cpp build script?"#, + ); - read_metadata(file) - .expect(r#" + read_metadata(file).expect( + r#" -- rust-cpp fatal error -- I/O error while reading metadata from target library file."#, @@ -89,8 +91,7 @@ NOTE: Double-check that the version of cpp_build and cpp_macros match"#, .collect::(); assert_eq!( - version, - VERSION, + version, VERSION, r#" -- rust-cpp fatal error -- @@ -108,7 +109,7 @@ Version mismatch between cpp_macros and cpp_build for same crate."# metadata .entry(hash) .or_insert(Vec::new()) - .push(MetaData{size, align, flags}); + .push(MetaData { size, align, flags }); } Ok(metadata) } @@ -234,7 +235,7 @@ NOTE: They cannot be generated by macro expansion."#, quote!(*const) }; - let arg_name : Ident = format!("arg_{}", written_name).into(); + let arg_name: Ident = format!("arg_{}", written_name).into(); extern_params.push(quote!(#arg_name : #ptr u8)); @@ -344,8 +345,10 @@ pub fn expand_wrap_class(input: TokenStream) -> TokenStream { ")", ",", "0", ")", ".", "1", "]", ",", "}" ]; - let s = source.find("stringify!(").expect("expected 'strignify!' token in class content") + 11; - let mut tokens : &str = &source[s..].trim(); + let s = source + .find("stringify!(") + .expect("expected 'strignify!' token in class content") + 11; + let mut tokens: &str = &source[s..].trim(); for token in SUFFIX.iter().rev() { assert!( @@ -357,8 +360,7 @@ pub fn expand_wrap_class(input: TokenStream) -> TokenStream { tokens = &tokens[..tokens.len() - token.len()].trim(); } - let class = parsing::cpp_class(synom::ParseState::new(tokens)) - .expect("cpp_class! macro"); + let class = parsing::cpp_class(synom::ParseState::new(tokens)).expect("cpp_class! macro"); let hash = class.name_hash(); @@ -374,16 +376,16 @@ NOTE: They cannot be generated by macro expansion."#, let (size, align) = (size_data[0].size, size_data[0].align); let base_type = match align { - 1 => { quote!(u8) } - 2 => { quote!(u16) } - 4 => { quote!(u32) } - 8 => { quote!(u64) } - _ => { panic!("unsupported alignment") } + 1 => quote!(u8), + 2 => quote!(u16), + 4 => quote!(u32), + 8 => quote!(u64), + _ => panic!("unsupported alignment"), }; - let destructor_name : Ident = format!("__cpp_destructor_{}", hash).into(); - let copyctr_name : Ident = format!("__cpp_copy_{}", hash).into(); - let defaultctr_name : Ident = format!("__cpp_default_{}", hash).into(); + let destructor_name: Ident = format!("__cpp_destructor_{}", hash).into(); + let copyctr_name: Ident = format!("__cpp_copy_{}", hash).into(); + let defaultctr_name: Ident = format!("__cpp_default_{}", hash).into(); let class_name = class.name.clone(); let mut result = quote! { From 22a6efd3563dde57f5b439e40f77448c00bd4f7f Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Thu, 30 Aug 2018 08:26:39 +0200 Subject: [PATCH 2/7] Port to syn 0.14 Porting to newer syn is required in order to support new rust connstruct. Fixes issue https://github.com/mystor/rust-cpp/issues/41 syn and proc_macro2 do not allow to get the line number or the verbatim captrued text: https://github.com/alexcrichton/proc-macro2/issues/110 This therefore has to redo the lexing manually to extract the relevant pieces for which we need these information. Only the more comples parsing is still done by syn. --- cpp_build/Cargo.toml | 7 +- cpp_build/src/lib.rs | 279 +++------------------ cpp_build/src/parser.rs | 537 ++++++++++++++++++++++++++++++++++++++++ cpp_build/src/strnom.rs | 421 +++++++++++++++++++++++++++++++ cpp_common/Cargo.toml | 6 +- cpp_common/src/lib.rs | 278 ++++++++------------- cpp_macros/Cargo.toml | 6 +- cpp_macros/src/lib.rs | 59 +++-- cpp_synmap/.gitignore | 2 - cpp_synmap/Cargo.toml | 16 -- cpp_synmap/README.md | 13 - cpp_synmap/src/lib.rs | 344 ------------------------- 12 files changed, 1126 insertions(+), 842 deletions(-) create mode 100644 cpp_build/src/parser.rs create mode 100644 cpp_build/src/strnom.rs delete mode 100644 cpp_synmap/.gitignore delete mode 100644 cpp_synmap/Cargo.toml delete mode 100644 cpp_synmap/README.md delete mode 100644 cpp_synmap/src/lib.rs diff --git a/cpp_build/Cargo.toml b/cpp_build/Cargo.toml index 9674601..cc6a00d 100644 --- a/cpp_build/Cargo.toml +++ b/cpp_build/Cargo.toml @@ -14,6 +14,7 @@ documentation = "https://docs.rs/cpp_build" lazy_static = "1.0" cc = "1.0" cpp_common = { path = "../cpp_common", version = "=0.4.0" } -cpp_synmap = { path = "../cpp_synmap", version = "0.3.0" } -cpp_syn = { version = "0.12", features=["full", "visit"] } -cpp_synom = { version = "0.12" } +syn = { version = "0.14", features=["full", "visit"] } +proc-macro2 = "0.4" +regex = "1" +unicode-xid = "0.1" diff --git a/cpp_build/src/lib.rs b/cpp_build/src/lib.rs index d292f7f..566dc50 100644 --- a/cpp_build/src/lib.rs +++ b/cpp_build/src/lib.rs @@ -4,28 +4,26 @@ //! For more information, see the [`cpp` crate module level //! documentation](https://docs.rs/cpp). -extern crate cpp_common; -extern crate cpp_syn as syn; -extern crate cpp_synom as synom; - -extern crate cpp_synmap; - extern crate cc; +extern crate cpp_common; +extern crate proc_macro2; +extern crate regex; +extern crate syn; +extern crate unicode_xid; #[macro_use] extern crate lazy_static; -use cpp_common::{ - flags, parsing, Capture, Class, Closure, ClosureSig, Macro, FILE_HASH, LIB_NAME, OUT_DIR, - STRUCT_METADATA_MAGIC, VERSION, -}; -use cpp_synmap::SourceMap; +#[macro_use] +mod strnom; + +use cpp_common::*; use std::env; use std::fs::{create_dir, remove_dir_all, File}; use std::io::prelude::*; use std::path::{Path, PathBuf}; -use syn::visit::Visitor; -use syn::{Ident, Mac, Spanned, Token, TokenTree, DUMMY_SPAN}; + +mod parser; fn warnln_impl(a: String) { for s in a.lines() { @@ -139,99 +137,15 @@ NOTE: rust-cpp's build function must be run in a build script."# )); } -enum ExpandSubMacroType<'a> { - Lit, - Closure(&'a mut u32), // the offset -} - -// Given a string containing some C++ code with a rust! macro, -// this functions expand the rust! macro to a call to an extern -// function -fn expand_sub_rust_macro(input: String, mut t: ExpandSubMacroType) -> String { - let mut result = input; - let mut extra_decl = String::new(); - loop { - let tmp = result.clone(); - if let synom::IResult::Done(_, rust_invocation) = - parsing::find_rust_macro(synom::ParseState::new(&tmp)) - { - let fn_name: Ident = match t { - ExpandSubMacroType::Lit => { - extra_decl.push_str(&format!("extern \"C\" void {}();\n", rust_invocation.id)); - rust_invocation.id.clone() - } - ExpandSubMacroType::Closure(ref mut offset) => { - **offset += 1; - format!( - "rust_cpp_callbacks{file_hash}[{offset}]", - file_hash = *FILE_HASH, - offset = **offset - 1 - ).into() - } - }; - - let mut decl_types = rust_invocation - .arguments - .iter() - .map(|&(_, ref val)| format!("rustcpp::argument_helper<{}>::type", val)) - .collect::>(); - let mut call_args = rust_invocation - .arguments - .iter() - .map(|&(ref val, _)| val.as_ref()) - .collect::>(); - - let fn_call = match rust_invocation.return_type { - None => format!( - "reinterpret_cast({f})({args})", - f = fn_name, - types = decl_types.join(", "), - args = call_args.join(", ") - ), - Some(rty) => { - decl_types.push(format!("rustcpp::return_helper<{rty}>", rty = rty)); - call_args.push("0"); - format!( - "std::move(*reinterpret_cast<{rty}*(*)({types})>({f})({args}))", - rty = rty, - f = fn_name, - types = decl_types.join(", "), - args = call_args.join(", ") - ) - } - }; - - let fn_call = { - // remove the rust! macro from the C++ snippet - let orig = result.drain(rust_invocation.begin..rust_invocation.end); - // add \ņ to the invocation in order to keep the same amount of line numbers - // so errors point to the right line. - orig.filter(|x| *x == '\n').fold(fn_call, |mut res, _| { - res.push('\n'); - res - }) - }; - // add the invocation of call where the rust! macro used to be. - result.insert_str(rust_invocation.begin, &fn_call); - } else { - break; - } - } - - return extra_decl + &result; -} - -fn gen_cpp_lib(visitor: &Handle) -> PathBuf { +fn gen_cpp_lib(visitor: &parser::Parser) -> PathBuf { let result_path = CPP_DIR.join("cpp_closures.cpp"); let mut output = File::create(&result_path).expect("Unable to generate temporary C++ file"); write!(output, "{}", INTERNAL_CPP_STRUCTS).unwrap(); if visitor.callbacks_count > 0 { - write!( - output, - add_line!( - r#" + #[cfg_attr(rustfmt, rustfmt_skip)] + write!(output, add_line!(r#" extern "C" {{ void (*rust_cpp_callbacks{file_hash}[{callbacks_count}])() = {{}}; }} @@ -306,10 +220,8 @@ extern "C" {{ .join(", "); if is_void { - write!( - output, - add_line!( - r#" + #[cfg_attr(rustfmt, rustfmt_skip)] + write!(output, add_line!(r#" extern "C" {{ void {name}({params}) {{ {body} @@ -319,19 +231,17 @@ void {name}({params}) {{ ), name = &name, params = params, - body = body.node + body = body ).unwrap(); } else { let comma = if params.is_empty() { "" } else { "," }; let args = captures .iter() - .map(|&Capture { ref name, .. }| name.as_ref()) + .map(|&Capture { ref name, .. }| name.to_string()) .collect::>() .join(", "); - write!( - output, - add_line!( - r#" + #[cfg_attr(rustfmt, rustfmt_skip)] + write!(output, add_line!(r#" static inline {ty} {name}_impl({params}) {{ {body} }} @@ -347,7 +257,7 @@ void {name}({params}{comma} void* __result) {{ comma = comma, ty = cpp, args = args, - body = body.node + body = body ).unwrap(); } } @@ -391,6 +301,7 @@ void {name}({params}{comma} void* __result) {{ magic.push(format!("{}", mag)); } + #[cfg_attr(rustfmt, rustfmt_skip)] write!(output, add_line!(r#" namespace rustcpp {{ @@ -705,35 +616,16 @@ impl Config { clean_artifacts(); // Parse the crate - let mut sm = SourceMap::new(); - let krate = match sm.add_crate_root(crate_root) { - Ok(krate) => krate, - Err(err) => { - let mut err_s = err.to_string(); - if let Some(i) = err_s.find("unparsed tokens after") { - // Strip the long error message from syn - err_s = err_s[0..i].to_owned(); - } - warnln!( - r#"-- rust-cpp parse error -- + let mut visitor = parser::Parser::default(); + if let Err(x) = visitor.parse_crate(&crate_root) { + warnln!(r#"-- rust-cpp parse error -- There was an error parsing the crate for the rust-cpp build script: {} In order to provide a better error message, the build script will exit successfully, such that rustc can provide an error message."#, - err_s + x ); - return; - } - }; - - // Parse the macro definitions - let mut visitor = Handle { - closures: Vec::new(), - classes: Vec::new(), - snippets: String::new(), - sm: &sm, - callbacks_count: 0, - }; - visitor.visit_crate(&krate); + return; + } // Generate the C++ library code let filename = gen_cpp_lib(&visitor); @@ -747,7 +639,6 @@ In order to provide a better error message, the build script will exit successfu if !self.std_flag_set { self.cc.flag_if_supported("-std=c++11"); } - // Build the C++ library self.cc.file(filename).compile(LIB_NAME); } @@ -758,117 +649,3 @@ In order to provide a better error message, the build script will exit successfu pub fn build>(path: P) { Config::new().build(path) } - -struct Handle<'a> { - closures: Vec, - classes: Vec, - snippets: String, - sm: &'a SourceMap, - callbacks_count: u32, -} - -fn line_directive(span: syn::Span, sm: &SourceMap) -> String { - let loc = sm.locinfo(span).unwrap(); - let mut line = format!("#line {} {:?}\n", loc.line, loc.path); - for _ in 0..loc.col { - line.push(' '); - } - return line; -} - -fn extract_with_span(spanned: &mut Spanned, src: &str, offset: usize, sm: &SourceMap) { - if spanned.span != DUMMY_SPAN { - let src_slice = &src[spanned.span.lo..spanned.span.hi]; - spanned.span.lo += offset; - spanned.span.hi += offset; - spanned.node = line_directive(spanned.span, sm); - spanned.node.push_str(src_slice); - } -} - -impl<'a> Visitor for Handle<'a> { - fn visit_mac(&mut self, mac: &Mac) { - if mac.path.segments.len() != 1 { - return; - } - if mac.path.segments[0].ident.as_ref() == "cpp" { - assert!(mac.tts.len() == 1); - self.handle_cpp(&mac.tts[0]); - } else if mac.path.segments[0].ident.as_ref() == "cpp_class" { - assert!(mac.tts.len() == 1); - self.handle_cpp_class(&mac.tts[0]); - } else { - self.parse_macro(&mac.tts); - } - } -} - -impl<'a> Handle<'a> { - fn handle_cpp(&mut self, tt: &TokenTree) { - let span = tt.span(); - let src = self.sm.source_text(span).unwrap(); - let input = synom::ParseState::new(&src); - match parsing::build_macro(input) - .expect(&format!("cpp! macro at {}", self.sm.locinfo(span).unwrap())) - { - Macro::Closure(mut c) => { - extract_with_span(&mut c.body, &src, span.lo, self.sm); - c.callback_offset = self.callbacks_count; - c.body.node = expand_sub_rust_macro( - c.body.node, - ExpandSubMacroType::Closure(&mut self.callbacks_count), - ); - self.closures.push(c); - } - Macro::Lit(mut l) => { - extract_with_span(&mut l, &src, span.lo, self.sm); - self.snippets.push('\n'); - self.snippets.push_str(&expand_sub_rust_macro( - l.node.clone(), - ExpandSubMacroType::Lit, - )); - } - } - } - - fn handle_cpp_class(&mut self, tt: &TokenTree) { - let span = tt.span(); - let src = self.sm.source_text(span).unwrap(); - let input = synom::ParseState::new(&src); - let mut class = parsing::class_macro(input).expect(&format!( - "cpp_class! macro at {}", - self.sm.locinfo(span).unwrap() - )); - class.line = line_directive(span, self.sm); - self.classes.push(class); - } - - fn parse_macro(&mut self, tts: &Vec) { - let mut last_ident: Option<&Ident> = None; - let mut is_macro = false; - for t in tts { - match t { - TokenTree::Token(Token::Not, _) => is_macro = true, - TokenTree::Token(Token::Ident(ref i), _) => { - is_macro = false; - last_ident = Some(&i); - } - TokenTree::Delimited(ref d, _) => { - if is_macro && last_ident.map_or(false, |i| i.as_ref() == "cpp") { - self.handle_cpp(&t) - } else if is_macro && last_ident.map_or(false, |i| i.as_ref() == "cpp_class") { - self.handle_cpp_class(&t) - } else { - self.parse_macro(&d.tts) - } - is_macro = false; - last_ident = None; - } - _ => { - is_macro = false; - last_ident = None; - } - } - } - } -} diff --git a/cpp_build/src/parser.rs b/cpp_build/src/parser.rs new file mode 100644 index 0000000..cf39427 --- /dev/null +++ b/cpp_build/src/parser.rs @@ -0,0 +1,537 @@ +use cpp_common::{parsing, Class, Closure, Macro}; +use regex::Regex; +use std::fmt; +use std::fs::File; +use std::io::Read; +use std::mem::swap; +use std::path::{Path, PathBuf}; +use syn; +use syn::visit::Visit; + +#[derive(Debug)] +pub enum Error { + ParseCannotOpenFile { + src_path: String, + }, + ParseSyntaxError { + src_path: String, + error: syn::synom::ParseError, + }, + LexError { + src_path: String, + line: u32, + }, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + &Error::ParseCannotOpenFile { ref src_path } => { + write!(f, "Parsing crate: cannot open file `{}`.", src_path) + } + &Error::ParseSyntaxError { + ref src_path, + ref error, + } => write!(f, "Parsing crate:`{}`:\n{:?}", src_path, error), + &Error::LexError { + ref src_path, + ref line, + } => write!(f, "{}:{}: Lexing error", src_path, line + 1), + } + } +} + +enum ExpandSubMacroType<'a> { + Lit, + Closure(&'a mut u32), // the offset +} + +// Given a string containing some C++ code with a rust! macro, +// this functions expand the rust! macro to a call to an extern +// function +fn expand_sub_rust_macro(input: String, mut t: ExpandSubMacroType) -> Result { + let mut result = input; + let mut extra_decl = String::new(); + let mut search_index = 0; + while let Some(index) = result[search_index..].find("rust!") { + let (rust_invocation, begin, end) = { + let mut cursor = new_cursor(&result); + cursor = cursor.advance(search_index + index); + search_index = cursor.off as usize + 4; + let end = find_delimited((find_delimited(cursor, "(")?.0).advance(1), ")")?.0; + let input = syn::buffer::TokenBuffer::new2( + (&cursor.rest[..(end.off + 1 - cursor.off) as usize]) + .parse() + .unwrap(), + ); + if let Ok(mut rust_invocation) = parsing::find_rust_macro(input.begin()) { + (rust_invocation.0, cursor.off as usize, end.off as usize + 1) + } else { + continue; + } + }; + let fn_name = match t { + ExpandSubMacroType::Lit => { + extra_decl.push_str(&format!("extern \"C\" void {}();\n", rust_invocation.id)); + rust_invocation.id.clone().to_string() + } + ExpandSubMacroType::Closure(ref mut offset) => { + use cpp_common::FILE_HASH; + **offset += 1; + format!( + "rust_cpp_callbacks{file_hash}[{offset}]", + file_hash = *FILE_HASH, + offset = **offset - 1 + ) + } + }; + + let mut decl_types = rust_invocation + .arguments + .iter() + .map(|&(_, ref val)| format!("rustcpp::argument_helper<{}>::type", val)) + .collect::>(); + let mut call_args = rust_invocation + .arguments + .iter() + .map(|&(ref val, _)| val.to_string()) + .collect::>(); + + let fn_call = match rust_invocation.return_type { + None => format!( + "reinterpret_cast({f})({args})", + f = fn_name, + types = decl_types.join(", "), + args = call_args.join(", ") + ), + Some(rty) => { + decl_types.push(format!("rustcpp::return_helper<{rty}>", rty = rty)); + call_args.push("0".to_string()); + format!( + "std::move(*reinterpret_cast<{rty}*(*)({types})>({f})({args}))", + rty = rty, + f = fn_name, + types = decl_types.join(", "), + args = call_args.join(", ") + ) + } + }; + + let fn_call = { + // remove the rust! macro from the C++ snippet + let orig = result.drain(begin..end); + // add \ņ to the invocation in order to keep the same amount of line numbers + // so errors point to the right line. + orig.filter(|x| *x == '\n').fold(fn_call, |mut res, _| { + res.push('\n'); + res + }) + }; + // add the invocation of call where the rust! macro used to be. + result.insert_str(begin, &fn_call); + } + + return Ok(extra_decl + &result); +} + +#[test] +fn test_expand_sub_rust_macro() { + let x = expand_sub_rust_macro( + "{ rust!(xxx [] { 1 }); }".to_owned(), + ExpandSubMacroType::Lit, + ); + assert_eq!( + x.unwrap(), + "extern \"C\" void xxx();\n{ reinterpret_cast(xxx)(); }" + ); + + let x = expand_sub_rust_macro( + "{ hello( rust!(xxx [] { 1 }), rust!(yyy [] { 2 }); ) }".to_owned(), + ExpandSubMacroType::Lit, + ); + assert_eq!(x.unwrap(), "extern \"C\" void xxx();\nextern \"C\" void yyy();\n{ hello( reinterpret_cast(xxx)(), reinterpret_cast(yyy)(); ) }"); +} + +#[path = "strnom.rs"] +mod strnom; +use strnom::*; + +fn skip_literal(mut input: Cursor) -> PResult { + //input = whitespace(input)?.0; + if input.starts_with("\"") { + input = cooked_string(input.advance(1))?.0; + debug_assert!(input.starts_with("\"")); + return Ok((input.advance(1), true)); + } + if input.starts_with("b\"") { + input = cooked_byte_string(input.advance(2))?.0; + debug_assert!(input.starts_with("\"")); + return Ok((input.advance(1), true)); + } + if input.starts_with("\'") { + input = input.advance(1); + let cur = cooked_char(input)?.0; + if !cur.starts_with("\'") { + return Ok((symbol(input)?.0, true)); + } + return Ok((cur.advance(1), true)); + } + if input.starts_with("b\'") { + input = cooked_byte(input.advance(2))?.0; + if !input.starts_with("\'") { + return Err(LexError { line: input.line }); + } + return Ok((input.advance(1), true)); + } + lazy_static! { + static ref RAW: Regex = Regex::new(r##"^b?r#*""##).unwrap(); + } + if RAW.is_match(input.rest) { + let q = input.rest.find('r').unwrap(); + input = input.advance(q + 1); + return raw_string(input).map(|x| (x.0, true)); + } + return Ok((input, false)); +} + +fn new_cursor(s: &str) -> Cursor { + Cursor { + rest: s, + off: 0, + line: 0, + column: 0, + } +} + +#[test] +fn test_skip_literal() -> Result<(), LexError> { + assert!((skip_literal(new_cursor(r#""fofofo"ok xx"#))?.0).starts_with("ok")); + assert!((skip_literal(new_cursor(r#""kk\"kdk"ok xx"#))?.0).starts_with("ok")); + assert!((skip_literal(new_cursor("r###\"foo \" bar \\\" \"###ok xx"))?.0).starts_with("ok")); + assert!( + (skip_literal(new_cursor("br###\"foo 'jjk' \" bar \\\" \"###ok xx"))?.0).starts_with("ok") + ); + assert!((skip_literal(new_cursor("'4'ok xx"))?.0).starts_with("ok")); + assert!((skip_literal(new_cursor("'\''ok xx"))?.0).starts_with("ok")); + assert!((skip_literal(new_cursor("b'\''ok xx"))?.0).starts_with("ok")); + assert!((skip_literal(new_cursor("'abc ok xx"))?.0).starts_with(" ok")); + assert!((skip_literal(new_cursor("'a ok xx"))?.0).starts_with(" ok")); + + assert!((skip_whitespace(new_cursor("ok xx"))).starts_with("ok")); + assert!((skip_whitespace(new_cursor(" ok xx"))).starts_with("ok")); + assert!( + (skip_whitespace(new_cursor( + " \n /* /*dd \n // */ */ // foo \n ok xx/* */" + ))).starts_with("ok") + ); + + Ok(()) +} + +// advance the cursor until it finds the needle. +fn find_delimited<'a>(mut input: Cursor<'a>, needle: &str) -> PResult<'a, ()> { + let mut stack: Vec<&'static str> = vec![]; + while !input.is_empty() { + input = skip_whitespace(input); + input = skip_literal(input)?.0; + if input.is_empty() { + break; + } + if stack.is_empty() && input.starts_with(needle) { + return Ok((input, ())); + } else if stack.last().map_or(false, |x| input.starts_with(x)) { + stack.pop(); + } else if input.starts_with("(") { + stack.push(")"); + } else if input.starts_with("[") { + stack.push("]"); + } else if input.starts_with("{") { + stack.push("}"); + } else if input.starts_with(")") || input.starts_with("]") || input.starts_with("}") { + return Err(LexError { line: input.line }); + } + input = input.advance(1); + } + return Err(LexError { line: input.line }); +} + +#[test] +fn test_find_delimited() -> Result<(), LexError> { + assert!((find_delimited(new_cursor(" x f ok"), "f")?.0).starts_with("f ok")); + assert!((find_delimited(new_cursor(" {f} f ok"), "f")?.0).starts_with("f ok")); + assert!( + (find_delimited(new_cursor(" (f\")\" { ( ) } /* ) */ f ) f ok"), "f")?.0) + .starts_with("f ok") + ); + Ok(()) +} + +#[test] +fn test_cursor_advance() -> Result<(), LexError> { + assert_eq!(new_cursor("\n\n\n").advance(2).line, 2); + assert_eq!(new_cursor("\n \n\n").advance(2).line, 1); + assert_eq!(new_cursor("\n\n\n").advance(2).column, 0); + assert_eq!(new_cursor("\n \n\n").advance(2).column, 1); + + assert_eq!( + (find_delimited(new_cursor("\n/*\n \n */ ( \n ) /* */ f"), "f")?.0).line, + 4 + ); + assert_eq!( + (find_delimited(new_cursor("\n/*\n \n */ ( \n ) /* */ f"), "f")?.0).column, + 9 + ); + Ok(()) +} + +fn line_directive(path: &PathBuf, cur: Cursor) -> String { + let mut line = format!("#line {} {:?}\n", cur.line + 1, path); + for _ in 0..cur.column { + line.push(' '); + } + return line; +} + +#[derive(Default)] +pub struct Parser { + pub closures: Vec, + pub classes: Vec, + pub snippets: String, + pub callbacks_count: u32, + current_path: PathBuf, // The current file being parsed + mod_dir: PathBuf, +} + +impl Parser { + pub fn parse_crate>(&mut self, crate_root: P) -> Result<(), Error> { + self.parse_mod(crate_root) + } + + fn parse_mod>(&mut self, mod_path: P) -> Result<(), Error> { + let mut s = String::new(); + let mut f = File::open(&mod_path).map_err(|_| Error::ParseCannotOpenFile { + src_path: mod_path.as_ref().to_str().unwrap().to_owned(), + })?; + f.read_to_string(&mut s) + .map_err(|_| Error::ParseCannotOpenFile { + src_path: mod_path.as_ref().to_str().unwrap().to_owned(), + })?; + + let fi = syn::parse_file(&s).map_err(|x| Error::ParseSyntaxError { + src_path: "".to_owned(), + error: x, + })?; + + let mut current_path = mod_path.as_ref().into(); + let mut mod_dir = mod_path.as_ref().parent().unwrap().into(); + + swap(&mut self.current_path, &mut current_path); + swap(&mut self.mod_dir, &mut mod_dir); + + self.find_cpp_macros(&s)?; + self.visit_file(&fi); + + swap(&mut self.current_path, &mut current_path); + swap(&mut self.mod_dir, &mut mod_dir); + + Ok(()) + } + + /* + fn parse_macro(&mut self, tts: TokenStream) { + let mut last_ident: Option = None; + let mut is_macro = false; + for t in tts.into_iter() { + match t { + TokenTree::Punct(ref p) if p.as_char() == '!' => is_macro = true, + TokenTree::Ident(i) => { + is_macro = false; + last_ident = Some(i); + } + TokenTree::Group(d) => { + if is_macro && last_ident.as_ref().map_or(false, |i| i == "cpp") { + self.handle_cpp(&d.stream()) + } else if is_macro && last_ident.as_ref().map_or(false, |i| i == "cpp_class") { + self.handle_cpp_class(&d.stream()) + } else { + self.parse_macro(d.stream()) + } + is_macro = false; + last_ident = None; + } + _ => { + is_macro = false; + last_ident = None; + } + } + } + } + */ + + fn find_cpp_macros(&mut self, source: &str) -> Result<(), Error> { + let mut cursor = new_cursor(source); + while !cursor.is_empty() { + cursor = skip_whitespace(cursor); + let r = skip_literal(cursor).map_err(|e| self.lex_error(e))?; + cursor = r.0; + if r.1 { + continue; + } + if let Ok((cur, ident)) = symbol(cursor) { + cursor = cur; + if ident != "cpp" && ident != "cpp_class" { + continue; + } + cursor = skip_whitespace(cursor); + if !cursor.starts_with("!") { + continue; + } + cursor = skip_whitespace(cursor.advance(1)); + let delim = if cursor.starts_with("(") { + ")" + } else if cursor.starts_with("[") { + "]" + } else if cursor.starts_with("{") { + "}" + } else { + continue; + }; + cursor = cursor.advance(1); + let mut macro_cur = cursor; + cursor = find_delimited(cursor, delim) + .map_err(|e| self.lex_error(e))? + .0; + let size = (cursor.off - macro_cur.off) as usize; + macro_cur.rest = ¯o_cur.rest[..size]; + if ident == "cpp" { + self.handle_cpp(macro_cur)?; + } else { + debug_assert_eq!(ident, "cpp_class"); + self.handle_cpp_class(macro_cur)?; + } + continue; + } + if cursor.is_empty() { + break; + } + cursor = cursor.advance(1); // Not perfect, but should work + } + Ok(()) + } + + fn lex_error(&self, e: LexError) -> Error { + Error::LexError { + src_path: self.current_path.clone().to_str().unwrap().to_owned(), + line: e.line, + } + } + + fn lex_error2(&self) -> Error { + Error::LexError { + src_path: self.current_path.clone().to_str().unwrap().to_owned(), + line: 0, + } + } + + fn handle_cpp(&mut self, x: Cursor) -> Result<(), Error> { + // Since syn don't give the exact string, we extract manually + let begin = (find_delimited(x, "{").map_err(|e| self.lex_error(e))?.0).advance(1); + let end = find_delimited(begin, "}").map_err(|e| self.lex_error(e))?.0; + let extracted = &begin.rest[..(end.off - begin.off) as usize]; + let extracted = line_directive(&self.current_path, begin) + extracted; + + let input = syn::buffer::TokenBuffer::new2(x.rest.parse().map_err(|_| self.lex_error2())?); + match parsing::build_macro(input.begin()) + .expect(&format!("cpp! macro at {:?}:{}", self.current_path, x.line)) + .0 + { + Macro::Closure(mut c) => { + c.callback_offset = self.callbacks_count; + c.body = expand_sub_rust_macro( + extracted.to_string(), + ExpandSubMacroType::Closure(&mut self.callbacks_count), + ).map_err(|e| self.lex_error(e))?; + self.closures.push(c); + } + Macro::Lit(_l) => { + self.snippets.push('\n'); + let snip = expand_sub_rust_macro(extracted.to_string(), ExpandSubMacroType::Lit) + .map_err(|e| self.lex_error(e))?; + self.snippets.push_str(&snip); + } + } + Ok(()) + } + + fn handle_cpp_class(&mut self, x: Cursor) -> Result<(), Error> { + let input = syn::buffer::TokenBuffer::new2(x.rest.parse().map_err(|_| self.lex_error2())?); + let mut class = parsing::cpp_class(input.begin()) + .expect(&format!( + "cpp_class! macro at {:?}:{}", + self.current_path, x.line + )) + .0; + class.line = line_directive(&self.current_path, x); + self.classes.push(class); + Ok(()) + } +} + +impl<'ast> Visit<'ast> for Parser { + /* This is currently commented out because proc_macro2 don't allow us to get the text verbatim + (https://github.com/alexcrichton/proc-macro2/issues/110#issuecomment-411959999) + fn visit_macro(&mut self, mac: &syn::Macro) { + if mac.path.segments.len() != 1 { + return; + } + if mac.path.segments[0].ident == "cpp" { + self.handle_cpp(&mac.tts); + } else if mac.path.segments[0].ident == "cpp_class" { + self.handle_cpp_class(&mac.tts); + } else { + self.parse_macro(mac.tts.clone()); + } + }*/ + + fn visit_item_mod(&mut self, item: &'ast syn::ItemMod) { + if item.content.is_some() { + let mut parent = self.mod_dir.join(item.ident.to_string()); + swap(&mut self.mod_dir, &mut parent); + syn::visit::visit_item_mod(self, item); + swap(&mut self.mod_dir, &mut parent); + return; + } + + // Determine the path of the inner module's file + for attr in &item.attrs { + match attr.interpret_meta() { + Some(syn::Meta::NameValue(syn::MetaNameValue { + ident: ref id, + lit: syn::Lit::Str(ref s), + .. + })) if id == "path" => + { + let mod_path = self.mod_dir.join(&s.value()); + return self.parse_mod(mod_path).unwrap(); + } + _ => {} + } + } + + let mod_name = item.ident.to_string(); + let mut subdir = self.mod_dir.join(mod_name.clone()); + subdir.push("mod.rs"); + if subdir.is_file() { + return self.parse_mod(subdir).unwrap(); + } + let adjacent = self.mod_dir.join(&format!("{}.rs", mod_name)); + if adjacent.is_file() { + return self.parse_mod(adjacent).unwrap(); + } + + panic!( + "No file with module definition for `mod {}` in file {:?}", + mod_name, self.current_path + ); + } +} diff --git a/cpp_build/src/strnom.rs b/cpp_build/src/strnom.rs new file mode 100644 index 0000000..23088dc --- /dev/null +++ b/cpp_build/src/strnom.rs @@ -0,0 +1,421 @@ +//! Adapted from [`nom`](https://github.com/Geal/nom). + +#![allow(dead_code)] // Why is this needed ? + +use std::str::{Bytes, CharIndices, Chars}; + +use unicode_xid::UnicodeXID; + +#[derive(Debug)] +pub struct LexError { + pub line: u32, +} + +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct Cursor<'a> { + pub rest: &'a str, + pub off: u32, + pub line: u32, + pub column: u32, +} + +impl<'a> Cursor<'a> { + pub fn advance(&self, amt: usize) -> Cursor<'a> { + let mut column_start: Option = None; + Cursor { + rest: &self.rest[amt..], + off: self.off + (amt as u32), + line: self.line + + self.rest[..amt] + .char_indices() + .filter(|&(_, ref x)| *x == '\n') + .map(|(i, _)| column_start = Some(i)) + .count() as u32, + column: match column_start { + None => self.column + (amt as u32), + Some(i) => (amt - i) as u32 - 1, + }, + } + } + + pub fn find(&self, p: char) -> Option { + self.rest.find(p) + } + + pub fn starts_with(&self, s: &str) -> bool { + self.rest.starts_with(s) + } + + pub fn is_empty(&self) -> bool { + self.rest.is_empty() + } + + pub fn len(&self) -> usize { + self.rest.len() + } + + pub fn as_bytes(&self) -> &'a [u8] { + self.rest.as_bytes() + } + + pub fn bytes(&self) -> Bytes<'a> { + self.rest.bytes() + } + + pub fn chars(&self) -> Chars<'a> { + self.rest.chars() + } + + pub fn char_indices(&self) -> CharIndices<'a> { + self.rest.char_indices() + } +} + +pub type PResult<'a, O> = Result<(Cursor<'a>, O), LexError>; + +pub fn whitespace(input: Cursor) -> PResult<()> { + if input.is_empty() { + return Err(LexError { line: input.line }); + } + + let bytes = input.as_bytes(); + let mut i = 0; + while i < bytes.len() { + let s = input.advance(i); + if bytes[i] == b'/' { + if s.starts_with("//") + // && (!s.starts_with("///") || s.starts_with("////")) + // && !s.starts_with("//!") + { + if let Some(len) = s.find('\n') { + i += len + 1; + continue; + } + break; + } else if s.starts_with("/**/") { + i += 4; + continue; + } else if s.starts_with("/*") + // && (!s.starts_with("/**") || s.starts_with("/***")) + // && !s.starts_with("/*!") + { + let (_, com) = block_comment(s)?; + i += com.len(); + continue; + } + } + match bytes[i] { + b' ' | 0x09...0x0d => { + i += 1; + continue; + } + b if b <= 0x7f => {} + _ => { + let ch = s.chars().next().unwrap(); + if is_whitespace(ch) { + i += ch.len_utf8(); + continue; + } + } + } + return if i > 0 { + Ok((s, ())) + } else { + Err(LexError { line: s.line }) + }; + } + Ok((input.advance(input.len()), ())) +} + +pub fn block_comment(input: Cursor) -> PResult<&str> { + if !input.starts_with("/*") { + return Err(LexError { line: input.line }); + } + + let mut depth = 0; + let bytes = input.as_bytes(); + let mut i = 0; + let upper = bytes.len() - 1; + while i < upper { + if bytes[i] == b'/' && bytes[i + 1] == b'*' { + depth += 1; + i += 1; // eat '*' + } else if bytes[i] == b'*' && bytes[i + 1] == b'/' { + depth -= 1; + if depth == 0 { + return Ok((input.advance(i + 2), &input.rest[..i + 2])); + } + i += 1; // eat '/' + } + i += 1; + } + Err(LexError { line: input.line }) +} + +pub fn skip_whitespace(input: Cursor) -> Cursor { + match whitespace(input) { + Ok((rest, _)) => rest, + Err(_) => input, + } +} + +fn is_whitespace(ch: char) -> bool { + // Rust treats left-to-right mark and right-to-left mark as whitespace + ch.is_whitespace() || ch == '\u{200e}' || ch == '\u{200f}' +} + +////// + +#[inline] +fn is_ident_start(c: char) -> bool { + ('a' <= c && c <= 'z') + || ('A' <= c && c <= 'Z') + || c == '_' + || (c > '\x7f' && UnicodeXID::is_xid_start(c)) +} + +#[inline] +fn is_ident_continue(c: char) -> bool { + ('a' <= c && c <= 'z') + || ('A' <= c && c <= 'Z') + || c == '_' + || ('0' <= c && c <= '9') + || (c > '\x7f' && UnicodeXID::is_xid_continue(c)) +} + +pub fn symbol(input: Cursor) -> PResult<&str> { + let mut chars = input.char_indices(); + + let raw = input.starts_with("r#"); + if raw { + chars.next(); + chars.next(); + } + + match chars.next() { + Some((_, ch)) if is_ident_start(ch) => {} + _ => return Err(LexError { line: input.line }), + } + + let mut end = input.len(); + for (i, ch) in chars { + if !is_ident_continue(ch) { + end = i; + break; + } + } + + let a = &input.rest[..end]; + if a == "r#_" { + Err(LexError { line: input.line }) + } else { + let ident = if raw { &a[2..] } else { a }; + Ok((input.advance(end), ident)) + } +} + +pub fn cooked_string(input: Cursor) -> PResult<()> { + let mut chars = input.char_indices().peekable(); + while let Some((byte_offset, ch)) = chars.next() { + match ch { + '"' => { + return Ok((input.advance(byte_offset), ())); + } + '\r' => { + if let Some((_, '\n')) = chars.next() { + // ... + } else { + break; + } + } + '\\' => match chars.next() { + Some((_, 'x')) => { + if !backslash_x_char(&mut chars) { + break; + } + } + Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\')) + | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {} + Some((_, 'u')) => { + if !backslash_u(&mut chars) { + break; + } + } + Some((_, '\n')) | Some((_, '\r')) => { + while let Some(&(_, ch)) = chars.peek() { + if ch.is_whitespace() { + chars.next(); + } else { + break; + } + } + } + _ => break, + }, + _ch => {} + } + } + Err(LexError { line: input.line }) +} + +pub fn cooked_byte_string(mut input: Cursor) -> PResult<()> { + let mut bytes = input.bytes().enumerate(); + 'outer: while let Some((offset, b)) = bytes.next() { + match b { + b'"' => { + return Ok((input.advance(offset), ())); + } + b'\r' => { + if let Some((_, b'\n')) = bytes.next() { + // ... + } else { + break; + } + } + b'\\' => match bytes.next() { + Some((_, b'x')) => { + if !backslash_x_byte(&mut bytes) { + break; + } + } + Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\')) + | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {} + Some((newline, b'\n')) | Some((newline, b'\r')) => { + let rest = input.advance(newline + 1); + for (offset, ch) in rest.char_indices() { + if !ch.is_whitespace() { + input = rest.advance(offset); + bytes = input.bytes().enumerate(); + continue 'outer; + } + } + break; + } + _ => break, + }, + b if b < 0x80 => {} + _ => break, + } + } + Err(LexError { line: input.line }) +} + +pub fn raw_string(input: Cursor) -> PResult<()> { + let mut chars = input.char_indices(); + let mut n = 0; + while let Some((byte_offset, ch)) = chars.next() { + match ch { + '"' => { + n = byte_offset; + break; + } + '#' => {} + _ => return Err(LexError { line: input.line }), + } + } + for (byte_offset, ch) in chars { + match ch { + '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => { + let rest = input.advance(byte_offset + 1 + n); + return Ok((rest, ())); + } + '\r' => {} + _ => {} + } + } + Err(LexError { line: input.line }) +} + +pub fn cooked_byte(input: Cursor) -> PResult<()> { + let mut bytes = input.bytes().enumerate(); + let ok = match bytes.next().map(|(_, b)| b) { + Some(b'\\') => match bytes.next().map(|(_, b)| b) { + Some(b'x') => backslash_x_byte(&mut bytes), + Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'') + | Some(b'"') => true, + _ => false, + }, + b => b.is_some(), + }; + if ok { + match bytes.next() { + Some((offset, _)) => { + if input.chars().as_str().is_char_boundary(offset) { + Ok((input.advance(offset), ())) + } else { + Err(LexError { line: input.line }) + } + } + None => Ok((input.advance(input.len()), ())), + } + } else { + Err(LexError { line: input.line }) + } +} + +pub fn cooked_char(input: Cursor) -> PResult<()> { + let mut chars = input.char_indices(); + let ok = match chars.next().map(|(_, ch)| ch) { + Some('\\') => match chars.next().map(|(_, ch)| ch) { + Some('x') => backslash_x_char(&mut chars), + Some('u') => backslash_u(&mut chars), + Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => { + true + } + _ => false, + }, + ch => ch.is_some(), + }; + if ok { + match chars.next() { + Some((idx, _)) => Ok((input.advance(idx), ())), + None => Ok((input.advance(input.len()), ())), + } + } else { + Err(LexError { line: input.line }) + } +} + +macro_rules! next_ch { + ($chars:ident @ $pat:pat $(| $rest:pat)*) => { + match $chars.next() { + Some((_, ch)) => match ch { + $pat $(| $rest)* => ch, + _ => return false, + }, + None => return false + } + }; +} + +fn backslash_x_char(chars: &mut I) -> bool +where + I: Iterator, +{ + next_ch!(chars @ '0'...'7'); + next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F'); + true +} + +fn backslash_x_byte(chars: &mut I) -> bool +where + I: Iterator, +{ + next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F'); + next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F'); + true +} + +fn backslash_u(chars: &mut I) -> bool +where + I: Iterator, +{ + next_ch!(chars @ '{'); + next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F'); + loop { + let c = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '_' | '}'); + if c == '}' { + return true; + } + } +} diff --git a/cpp_common/Cargo.toml b/cpp_common/Cargo.toml index 593b14a..b4a7862 100644 --- a/cpp_common/Cargo.toml +++ b/cpp_common/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/mystor/rust-cpp" documentation = "https://docs.rs/cpp_common" [dependencies] +syn = { version = "0.14", features = ["full", "visit", "extra-traits"] } lazy_static = "1.0" -cpp_syn = { version = "0.12", features = ["full", "visit"] } -cpp_synom = { version = "0.12" } -quote = "0.3" +quote = "0.6" +proc-macro2 = "0.4" diff --git a/cpp_common/src/lib.rs b/cpp_common/src/lib.rs index 148f462..ec60f56 100644 --- a/cpp_common/src/lib.rs +++ b/cpp_common/src/lib.rs @@ -1,9 +1,6 @@ #[macro_use] -extern crate cpp_syn as syn; - -#[macro_use] -extern crate cpp_synom as synom; - +extern crate syn; +extern crate proc_macro2; #[macro_use] extern crate quote; @@ -15,7 +12,8 @@ use std::env; use std::hash::{Hash, Hasher}; use std::path::PathBuf; -use syn::{Ident, MetaItem, Spanned, Ty}; +use proc_macro2::{Span, TokenTree}; +use syn::{Attribute, Ident, Type}; pub const VERSION: &'static str = env!("CARGO_PKG_VERSION"); @@ -75,7 +73,7 @@ pub struct Capture { #[derive(Clone, Debug, Hash, PartialEq, Eq)] pub struct ClosureSig { pub captures: Vec, - pub ret: Ty, + pub ret: Type, pub cpp: String, pub std_body: String, } @@ -89,14 +87,17 @@ impl ClosureSig { } pub fn extern_name(&self) -> Ident { - format!("__cpp_closure_{}", self.name_hash()).into() + Ident::new( + &format!("__cpp_closure_{}", self.name_hash()), + Span::call_site(), + ) } } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug)] pub struct Closure { pub sig: ClosureSig, - pub body: Spanned, + pub body: String, pub callback_offset: u32, } @@ -104,8 +105,7 @@ pub struct Closure { pub struct Class { pub name: Ident, pub cpp: String, - pub public: bool, - pub attrs: Vec, + pub attrs: Vec, pub line: String, // the #line directive } @@ -114,114 +114,82 @@ impl Class { let mut hasher = DefaultHasher::new(); self.name.hash(&mut hasher); self.cpp.hash(&mut hasher); - self.public.hash(&mut hasher); hasher.finish() } pub fn derives(&self, i: &str) -> bool { self.attrs.iter().any(|x| { - if let MetaItem::List(ref n, ref list) = x { - n.as_ref() == "derive" && list.iter().any(|y| { - if let syn::NestedMetaItem::MetaItem(MetaItem::Word(ref d)) = y { - d.as_ref() == i - } else { - false - } - }) - } else { - false - } + use syn::{Meta, NestedMeta}; + x.interpret_meta().map_or(false, |m| { + if let Meta::List(ref list) = m { + list.ident == "derive" && list.nested.iter().any(|y| { + if let NestedMeta::Meta(Meta::Word(ref d)) = y { + d == i + } else { + false + } + }) + } else { + false + } + }) }) } } +#[derive(Debug)] pub enum Macro { Closure(Closure), - Lit(Spanned), + Lit(TokenTree), } +#[derive(Debug)] pub struct RustInvocation { - pub begin: usize, - pub end: usize, + pub begin: Span, + pub end: Span, pub id: Ident, pub return_type: Option, - pub arguments: Vec<(String, String)>, // Vec of name and type + pub arguments: Vec<(Ident, String)>, // Vec of name and type } pub mod parsing { use super::{Capture, Class, Closure, ClosureSig, Macro, RustInvocation}; - use syn::parse::{ident, lit, string, tt, ty}; - use syn::{Ident, MetaItem, NestedMetaItem, Spanned, Ty, DUMMY_SPAN}; - use synom::space::{block_comment, whitespace}; - - macro_rules! mac_body { - ($i: expr, $submac:ident!( $($args:tt)* )) => { - delimited!( - $i, - alt!(punct!("[") | punct!("(") | punct!("{")), - $submac!($($args)*), - alt!(punct!("]") | punct!(")") | punct!("}")) - ) - }; - ($i:expr, $e:expr) => { - mac_body!($i, call!($e)) - }; - } + use proc_macro2::TokenTree; + use syn::{punctuated::Punctuated, Attribute, Ident, LitStr, Type, Visibility}; - named!(ident_or_self -> Ident, alt!( ident | keyword!("self") => { |_| "self".into() } )); + named!(ident_or_self -> Ident, alt!(syn!(Ident) | + keyword!(self) => { |x| x.into() } )); named!(name_as_string -> Capture, do_parse!( - is_mut: option!(keyword!("mut")) >> + is_mut: option!(keyword!(mut)) >> id: ident_or_self >> - keyword!("as") >> - cty: string >> + keyword!(as) >> + cty: syn!(LitStr) >> (Capture { mutable: is_mut.is_some(), name: id, - cpp: cty.value + cpp: cty.value() }) )); - named!(captures -> Vec, - delimited!( - punct!("["), - terminated_list!( - punct!(","), - name_as_string - ), - punct!("]"))); + named!(captures -> Vec, map!(brackets!(call!( + Punctuated::::parse_separated_with, name_as_string)), |x| x.1.into_iter().collect())); - named!(ret_ty -> (Ty, String), + named!(ret_ty -> (Type, String), alt!( - do_parse!(punct!("->") >> - rty: ty >> - keyword!("as") >> - cty: string >> - ((rty, cty.value))) | - value!((Ty::Tup(Vec::new()), "void".to_owned())) + do_parse!(punct!(->) >> + rty: syn!(Type) >> + keyword!(as) >> + cty: syn!(LitStr) >> + ((rty, cty.value()))) | + value!((parse_quote!{()}, "void".to_owned())) )); - named!(code_block -> Spanned, - alt!( - do_parse!(s: string >> (Spanned { - node: s.value, - span: DUMMY_SPAN, - })) | - // XXX: This is really inefficient and means that things like ++y - // will parse incorrectly (as we care about the layout of the - // original source) so consider this a temporary monkey patch. - // Once we get spans we can work past it. - delimited!(punct!("{"), spanned!(many0!(tt)), punct!("}")) => { - |Spanned{ node, span }| Spanned { - node: quote!(#(#node)*).to_string(), - span: span - } - } - )); + named!(code_block -> TokenTree, syn!(TokenTree)); named!(pub cpp_closure -> Closure, - do_parse!(option!(keyword!("unsafe")) >> + do_parse!(option!(keyword!(unsafe)) >> captures: captures >> ret: ret_ty >> code: code_block >> @@ -230,127 +198,75 @@ pub mod parsing { captures: captures, ret: ret.0, cpp: ret.1, - std_body: code.node.clone(), + // Need to filter the spaces because there is a difference between + // proc_macro2 and proc_macro and the hashes would not match + std_body: code.to_string().chars().filter(|x| *x != ' ').collect(), }, - body: code, + body: code.to_string(), callback_offset: 0 }))); - named!(pub build_macro -> Macro , mac_body!(alt!( + named!(pub build_macro -> Macro , alt!( cpp_closure => { |c| Macro::Closure(c) } | code_block => { |b| Macro::Lit(b) } - ))); - - named!(pub expand_macro -> Closure, mac_body!( - map!(tuple!( - punct!("@"), keyword!("TYPE"), cpp_closure - ), (|(_, _, x)| x)))); - - //FIXME: make cpp_syn::attr::parsing::outer_attr public - // This is just a trimmed down version of it - named!(pub outer_attr -> MetaItem, alt!( - do_parse!( - punct!("#") >> - punct!("[") >> - attr: meta_item >> - punct!("]") >> - (attr) - ) | do_parse!( - punct!("///") >> - not!(tag!("/")) >> - content: spanned!(take_until!("\n")) >> - (MetaItem::NameValue("doc".into(), content.node.into())) - ) | do_parse!( - option!(whitespace) >> - peek!(tuple!(tag!("/**"), not!(tag!("*")))) >> - com: block_comment >> - (MetaItem::NameValue("doc".into(), com[3..com.len()-2].into())) - ) - )); - - named!(meta_item -> MetaItem, alt!( - do_parse!( - id: ident >> - punct!("(") >> - inner: terminated_list!(punct!(","), nested_meta_item) >> - punct!(")") >> - (MetaItem::List(id, inner)) - ) - | - do_parse!( - name: ident >> - punct!("=") >> - value: lit >> - (MetaItem::NameValue(name, value)) - ) - | - map!(ident, MetaItem::Word) - )); - named!(nested_meta_item -> NestedMetaItem, alt!( - meta_item => { NestedMetaItem::MetaItem } - | - lit => { NestedMetaItem::Literal } )); named!(pub cpp_class -> Class, do_parse!( - attrs: many0!(outer_attr) >> - is_pub: option!(tuple!( - keyword!("pub"), - option!(delimited!(punct!("("), many0!(tt), punct!(")"))))) >> - keyword!("unsafe") >> - keyword!("struct") >> - name: ident >> - keyword!("as") >> - cpp_type: string >> + attrs: many0!(Attribute::parse_outer) >> + option!(syn!(Visibility)) >> + keyword!(unsafe) >> + keyword!(struct) >> + name: syn!(Ident) >> + keyword!(as) >> + cpp_type: syn!(LitStr) >> (Class { name: name, - cpp: cpp_type.value, - public: is_pub.is_some(), + cpp: cpp_type.value(), attrs: attrs, line: String::default(), - }))); - named!(pub class_macro -> Class , mac_body!(cpp_class)); + }))); - named!(rust_macro_argument -> (String, String), + named!(rust_macro_argument -> (Ident, String), do_parse!( - name: ident >> - punct!(":") >> - ty >> - keyword!("as") >> - cty: string >> - ((name.as_ref().to_owned(), cty.value)))); + name: syn!(Ident) >> + punct!(:) >> + syn!(Type) >> + keyword!(as) >> + cty: syn!(LitStr) >> + ((name, cty.value())))); named!(pub find_rust_macro -> RustInvocation, do_parse!( - alt!(take_until!("rust!") | take_until!("rust !")) >> - begin: spanned!(keyword!("rust")) >> - punct!("!") >> - punct!("(") >> - id: ident >> - punct!("[") >> - args: separated_list!(punct!(","), rust_macro_argument) >> - punct!("]") >> - rty : option!(do_parse!(punct!("->") >> - ty >> - keyword!("as") >> - cty: string >> - (cty.value))) >> - tt >> - end: spanned!(punct!(")")) >> + begin: custom_keyword!(rust) >> + punct!(!) >> + content: parens!(do_parse!( + id: syn!(Ident) >> + args: map!(brackets!(call!(Punctuated::<(Ident, String), Token![,]>::parse_separated_with, rust_macro_argument)), + |x| x.1.into_iter().collect()) >> + rty : option!( + do_parse!(punct!(->) >> + syn!(Type) >> + keyword!(as) >> + cty: syn!(LitStr) >> + (cty.value()))) >> + syn!(TokenTree) >> + (id, args, rty))) >> (RustInvocation{ - begin: begin.span.lo, - end: end.span.hi, - id: id, - return_type: rty, - arguments: args + begin: begin.span(), + end: (content.0).0, + id: (content.1).0, + return_type: (content.1).2, + arguments: (content.1).1 }))); named!(pub find_all_rust_macro -> Vec, - do_parse!( - r : many0!(find_rust_macro) >> - many0!(alt!( tt => {|_| ""} | punct!("]") | punct!(")") | punct!("}"))) - >> (r))); + map!(many0!(alt!(find_rust_macro => {|x| vec![x] } + | map!(brackets!(call!(find_all_rust_macro)), |x| x.1) + | map!(parens!(call!(find_all_rust_macro)), |x| x.1) + | map!(braces!(call!(find_all_rust_macro)), |x| x.1) + | syn!(TokenTree) => { |_| vec![] })), + |x| x.into_iter().flat_map(|x|x).collect())); } diff --git a/cpp_macros/Cargo.toml b/cpp_macros/Cargo.toml index 3981458..b268113 100644 --- a/cpp_macros/Cargo.toml +++ b/cpp_macros/Cargo.toml @@ -16,8 +16,8 @@ proc-macro = true [dependencies] lazy_static = "1.0" cpp_common = { path = "../cpp_common", version = "=0.4.0" } -cpp_synom = { version = "0.12" } -cpp_syn = { version = "0.12", features=["full", "visit"] } -quote = "0.3" +syn = { version = "0.14", features=["full", "visit"] } +quote = "0.6" +proc-macro2 = "0.4" aho-corasick = "0.6" byteorder = "1.0" diff --git a/cpp_macros/src/lib.rs b/cpp_macros/src/lib.rs index 1a69733..fab00a5 100644 --- a/cpp_macros/src/lib.rs +++ b/cpp_macros/src/lib.rs @@ -5,9 +5,7 @@ //! documentation](https://docs.rs/cpp). #![recursion_limit = "128"] -extern crate cpp_synom as synom; - -extern crate cpp_syn as syn; +extern crate syn; #[macro_use] extern crate quote; @@ -15,6 +13,8 @@ extern crate quote; extern crate cpp_common; extern crate proc_macro; +extern crate proc_macro2; +use proc_macro2::Span; #[macro_use] extern crate lazy_static; @@ -24,7 +24,6 @@ extern crate aho_corasick; extern crate byteorder; use cpp_common::{flags, parsing, FILE_HASH, LIB_NAME, MSVC_LIB_NAME, OUT_DIR, VERSION}; -use proc_macro::TokenStream; use std::collections::HashMap; use syn::Ident; @@ -166,7 +165,7 @@ fn macro_text(mut source: &str) -> &str { } #[proc_macro_derive(__cpp_internal_closure)] -pub fn expand_internal(input: TokenStream) -> TokenStream { +pub fn expand_internal(input: proc_macro::TokenStream) -> proc_macro::TokenStream { assert_eq!( env!("CARGO_PKG_VERSION"), VERSION, @@ -177,7 +176,8 @@ pub fn expand_internal(input: TokenStream) -> TokenStream { let source = input.to_string(); let tokens = macro_text(&source); - let closure = parsing::cpp_closure(synom::ParseState::new(tokens)).expect("cpp! macro"); + let input = syn::buffer::TokenBuffer::new(tokens.parse().unwrap()); + let closure = parsing::cpp_closure(input.begin()).expect("cpp! macro").0; // Get the size data compiled by the build macro let size_data = METADATA.get(&closure.sig.name_hash()).expect( @@ -194,8 +194,8 @@ NOTE: They cannot be generated by macro expansion."#, let mut call_args = Vec::new(); for (i, capture) in closure.sig.captures.iter().enumerate() { let written_name = &capture.name; - let mac_name: Ident = format!("$var_{}", written_name).into(); - let mac_cty: Ident = format!("$cty_{}", written_name).into(); + let mac_name = Ident::new(&format!("var_{}", written_name), Span::call_site()); + let mac_cty = Ident::new(&format!("cty_{}", written_name), Span::call_site()); // Generate the assertion to check that the size and align of the types // match before calling. @@ -215,12 +215,12 @@ NOTE: They cannot be generated by macro expansion."#, // a no-op. ::std::mem::forget( ::std::mem::transmute::<_, [u8; #size]>( - ::std::ptr::read(&#mac_name))); + ::std::ptr::read(&$#mac_name))); // NOTE: Both of these calls should be dead code in opt builds. - assert!(::std::mem::size_of_val(&#mac_name) == #size, + assert!(::std::mem::size_of_val(&$#mac_name) == #size, #sizeof_msg); - assert!(::std::mem::align_of_val(&#mac_name) == #align, + assert!(::std::mem::align_of_val(&$#mac_name) == #align, #alignof_msg); }; @@ -235,15 +235,15 @@ NOTE: They cannot be generated by macro expansion."#, quote!(*const) }; - let arg_name: Ident = format!("arg_{}", written_name).into(); + let arg_name = Ident::new(&format!("arg_{}", written_name), Span::call_site()); extern_params.push(quote!(#arg_name : #ptr u8)); - tt_args.push(quote!(#mb_mut #mac_name : ident as #mac_cty : tt)); + tt_args.push(quote!(#mb_mut $#mac_name : ident as $#mac_cty : tt)); call_args.push(quote!({ #assertion - &#mb_mut #mac_name as #ptr _ as #ptr u8 + &#mb_mut $#mac_name as #ptr _ as #ptr u8 })); } @@ -280,10 +280,15 @@ NOTE: They cannot be generated by macro expansion."#, } }; - let rust_invocations = parsing::find_all_rust_macro(synom::ParseState::new(&closure.body.node)) - .expect("rust! macro"); + let input = syn::buffer::TokenBuffer::new(closure.body.parse().unwrap()); + let rust_invocations = parsing::find_all_rust_macro(input.begin()) + .expect("rust! macro") + .0; let init_callbacks = if !rust_invocations.is_empty() { - let rust_cpp_callbacks: Ident = format!("rust_cpp_callbacks{}", *FILE_HASH).into(); + let rust_cpp_callbacks = Ident::new( + &format!("rust_cpp_callbacks{}", *FILE_HASH), + Span::call_site(), + ); let offset = (flags >> 32) as isize; let callbacks: Vec = rust_invocations.iter().map(|x| x.id.clone()).collect(); quote! { @@ -332,12 +337,11 @@ NOTE: They cannot be generated by macro expansion."#, } } }; - result.to_string().parse().unwrap() } #[proc_macro_derive(__cpp_internal_class)] -pub fn expand_wrap_class(input: TokenStream) -> TokenStream { +pub fn expand_wrap_class(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let source = input.to_string(); #[cfg_attr(rustfmt, rustfmt_skip)] @@ -360,7 +364,10 @@ pub fn expand_wrap_class(input: TokenStream) -> TokenStream { tokens = &tokens[..tokens.len() - token.len()].trim(); } - let class = parsing::cpp_class(synom::ParseState::new(tokens)).expect("cpp_class! macro"); + let input = syn::buffer::TokenBuffer::new(tokens.parse().unwrap()); + let class = parsing::cpp_class(input.begin()) + .expect("cpp_class! macro") + .0; let hash = class.name_hash(); @@ -383,9 +390,9 @@ NOTE: They cannot be generated by macro expansion."#, _ => panic!("unsupported alignment"), }; - let destructor_name: Ident = format!("__cpp_destructor_{}", hash).into(); - let copyctr_name: Ident = format!("__cpp_copy_{}", hash).into(); - let defaultctr_name: Ident = format!("__cpp_default_{}", hash).into(); + let destructor_name = Ident::new(&format!("__cpp_destructor_{}", hash), Span::call_site()); + let copyctr_name = Ident::new(&format!("__cpp_copy_{}", hash), Span::call_site()); + let defaultctr_name = Ident::new(&format!("__cpp_default_{}", hash), Span::call_site()); let class_name = class.name.clone(); let mut result = quote! { @@ -452,7 +459,7 @@ NOTE: They cannot be generated by macro expansion."#, } if class.derives("PartialEq") { - let equal_name: Ident = format!("__cpp_equal_{}", hash).into(); + let equal_name = Ident::new(&format!("__cpp_equal_{}", hash), Span::call_site()); result = quote!{ #result impl ::std::cmp::PartialEq for #class_name { fn eq(&self, other: &#class_name) -> bool { @@ -465,7 +472,7 @@ NOTE: They cannot be generated by macro expansion."#, }; } if class.derives("PartialOrd") { - let compare_name: Ident = format!("__cpp_compare_{}", hash).into(); + let compare_name = Ident::new(&format!("__cpp_compare_{}", hash), Span::call_site()); let f = |func, cmp| { quote!{ fn #func(&self, other: &#class_name) -> bool { @@ -500,7 +507,7 @@ NOTE: They cannot be generated by macro expansion."#, }; } if class.derives("Ord") { - let compare_name: Ident = format!("__cpp_compare_{}", hash).into(); + let compare_name = Ident::new(&format!("__cpp_compare_{}", hash), Span::call_site()); result = quote!{ #result impl ::std::cmp::Ord for #class_name { fn cmp(&self, other: &#class_name) -> ::std::cmp::Ordering { diff --git a/cpp_synmap/.gitignore b/cpp_synmap/.gitignore deleted file mode 100644 index a9d37c5..0000000 --- a/cpp_synmap/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -target -Cargo.lock diff --git a/cpp_synmap/Cargo.toml b/cpp_synmap/Cargo.toml deleted file mode 100644 index 6cb3e67..0000000 --- a/cpp_synmap/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "cpp_synmap" -version = "0.3.0" -authors = ["Nika Layzell "] -description = "Sourcemap and full crate parsing support for `cpp_syn`" -readme = "README.md" -license = "MIT/Apache-2.0" -keywords = ["syn", "parsing", "sourcemap", "crate"] -categories = ["development-tools::procedural-macro-helpers"] -repository = "https://github.com/mystor/rust-cpp" -documentation = "https://docs.rs/cpp_synmap" - -[dependencies] -memchr = "2.0" -cpp_syn = { version = "0.12", features = ["full", "fold"] } -cpp_synom = { version = "0.12" } diff --git a/cpp_synmap/README.md b/cpp_synmap/README.md deleted file mode 100644 index 591d6ec..0000000 --- a/cpp_synmap/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Synmap - -> NOTE: This crate currently depends on `cpp_syn` rather than `syn`, as it -> requires the `Span` features which have not been landed in `syn` yet for the -> majority of its features. - -This crate provides a `SourceMap` type which can be used to parse an entire -crate and generate a complete AST. It also updates the spans in the parsed AST -to be relative to the `SourceMap` rather than the bytes in the input file. - -With this information, the `SourceMap` provides methods to map spans to source -filenames (`filename`), source text (`source_text`) and line/column numbers -(`locinfo`). diff --git a/cpp_synmap/src/lib.rs b/cpp_synmap/src/lib.rs deleted file mode 100644 index 0d5390a..0000000 --- a/cpp_synmap/src/lib.rs +++ /dev/null @@ -1,344 +0,0 @@ -//! `synmap` provides utilities for parsing multi-file crates into `syn` AST -//! nodes, and resolving the spans attached to those nodes into raw source text, -//! and line/column information. -//! -//! The primary entry point for the crate is the `SourceMap` type, which stores -//! mappings from byte offsets to file information, along with cached file -//! information. - -extern crate cpp_syn as syn; - -extern crate memchr; - -use std::fmt::{self, Write}; -use std::error; -use std::fs::{self, File}; -use std::io::prelude::*; -use std::io::{self, Error, ErrorKind}; -use std::path::{Path, PathBuf}; -use syn::{Attribute, Crate, Ident, Item, ItemKind, Lit, LitKind, MetaItem, Span}; -use syn::fold::{self, Folder}; -use std::mem; - -/// This constant controls the amount of padding which is created between -/// consecutive files' span ranges. It is non-zero to ensure that the low byte -/// offset of one file is not equal to the high byte offset of the previous -/// file. -const FILE_PADDING_BYTES: usize = 1; - -/// Information regarding the on-disk location of a span of code. -/// This type is produced by `SourceMap::locinfo`. -#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] -pub struct LocInfo<'a> { - pub path: &'a Path, - pub line: usize, - pub col: usize, -} - -impl<'a> fmt::Display for LocInfo<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}:{}:{}", self.path.display(), self.line, self.col) - } -} - -#[derive(Debug)] -struct FileInfo { - span: Span, - path: PathBuf, - src: String, - lines: Vec, -} - -/// NOTE: This produces line and column. Line is 1-indexed, column is 0-indexed -fn offset_line_col(lines: &Vec, off: usize) -> (usize, usize) { - match lines.binary_search(&off) { - Ok(found) => (found + 1, 0), - Err(idx) => (idx, off - lines[idx - 1]), - } -} - -fn lines_offsets(s: &[u8]) -> Vec { - let mut lines = vec![0]; - let mut prev = 0; - while let Some(len) = memchr::memchr(b'\n', &s[prev..]) { - prev += len + 1; - lines.push(prev); - } - lines -} - -/// The `SourceMap` is the primary entry point for `synmap`. It maintains a -/// mapping between `Span` objects and the original source files they were -/// parsed from. -#[derive(Debug)] -pub struct SourceMap { - files: Vec, - offset: usize, -} - -impl SourceMap { - /// Create a new `SourceMap` object with no files inside of it. - pub fn new() -> SourceMap { - SourceMap { - files: Vec::new(), - offset: 0, - } - } - - /// Read and parse the passed-in file path as a crate root, recursively - /// parsing each of the submodules. Produces a syn `Crate` object with - /// all submodules inlined. - /// - /// `Span` objects inside the resulting crate object are `SourceMap` - /// relative, and should be interpreted by passing to the other methods on - /// this type, such as `locinfo`, `source_text`, or `filename`. - pub fn add_crate_root>(&mut self, path: P) -> io::Result { - self.parse_canonical_file(fs::canonicalize(path)?) - } - - /// This is an internal method which requires a canonical pathbuf as - /// returned from a method like `fs::canonicalize`. - fn parse_canonical_file(&mut self, path: PathBuf) -> io::Result { - // Parse the crate with syn - let mut source = String::new(); - File::open(&path)?.read_to_string(&mut source)?; - let krate = syn::parse_crate(&source).map_err(|e| Error::new(ErrorKind::InvalidData, e))?; - - let parent = path.parent() - .ok_or(Error::new( - ErrorKind::InvalidInput, - "cannot parse file without parent directory", - ))? - .to_path_buf(); - - // Register the read-in file in the SourceMap - let offset = self.offset; - self.offset += source.len() + FILE_PADDING_BYTES; - self.files.push(FileInfo { - span: Span { - lo: offset, - hi: offset + source.len(), - }, - path: path, - lines: lines_offsets(source.as_bytes()), - src: source, - }); - - // Walk the parsed Crate object, recursively filling in the bodies of - // `mod` statements, and rewriting spans to be SourceMap-relative - // instead of file-relative. - let idx = self.files.len() - 1; - let mut walker = Walker { - idx: idx, - error: None, - sm: self, - parent: parent, - }; - - let krate = walker.fold_crate(krate); - if let Some(err) = walker.error { - return Err(err); - } - - Ok(krate) - } - - - fn local_fileinfo(&self, mut span: Span) -> io::Result<(&FileInfo, Span)> { - if span.lo > span.hi { - return Err(Error::new( - ErrorKind::InvalidInput, - "Invalid span object with negative length", - )); - } - for fi in &self.files { - if span.lo >= fi.span.lo && span.lo <= fi.span.hi && span.hi >= fi.span.lo - && span.hi <= fi.span.hi - { - // Remove the offset - span.lo -= fi.span.lo; - span.hi -= fi.span.lo; - // Set the path - return Ok((fi, span)); - } - } - Err(Error::new( - ErrorKind::InvalidInput, - "Span is not part of any input file", - )) - } - - /// Get the filename which contains the given span. - /// - /// Fails if the span is invalid or spans multiple source files. - pub fn filename(&self, span: Span) -> io::Result<&Path> { - Ok(&self.local_fileinfo(span)?.0.path) - } - - /// Get the source text for the passed-in span. - /// - /// Fails if the span is invalid or spans multiple source files. - pub fn source_text(&self, span: Span) -> io::Result<&str> { - let (fi, span) = self.local_fileinfo(span)?; - Ok(&fi.src[span.lo..span.hi]) - } - - /// Get a LocInfo object for the passed-in span, containing line, column, - /// and file name information for the beginning and end of the span. The - /// `path` field in the returned LocInfo struct will be a reference to a - /// canonical path. - /// - /// Fails if the span is invalid or spans multiple source files. - pub fn locinfo(&self, span: Span) -> io::Result { - let (fi, span) = self.local_fileinfo(span)?; - - let (line, col) = offset_line_col(&fi.lines, span.lo); - Ok(LocInfo { - path: &fi.path, - line: line, - col: col, - }) - } -} - -struct Walker<'a> { - idx: usize, - error: Option, - sm: &'a mut SourceMap, - parent: PathBuf, -} - -impl<'a> Walker<'a> { - fn read_submodule(&mut self, path: PathBuf) -> io::Result { - let faux_crate = self.sm.parse_canonical_file(path)?; - if faux_crate.shebang.is_some() { - return Err(Error::new( - ErrorKind::InvalidData, - "Only the root file of a crate may contain shebangs", - )); - } - - Ok(faux_crate) - } - - fn get_attrs_items(&mut self, attrs: &[Attribute], ident: &Ident) -> io::Result { - // Determine the path of the inner module's file - for attr in attrs { - match attr.value { - MetaItem::NameValue( - ref id, - Lit { - node: LitKind::Str(ref s, _), - .. - }, - ) => if id.as_ref() == "path" { - let explicit = self.parent.join(&s[..]); - return self.read_submodule(explicit); - }, - _ => {} - } - } - - let mut subdir = self.parent.join(ident.as_ref()); - subdir.push("mod.rs"); - if subdir.is_file() { - return self.read_submodule(subdir); - } - - let adjacent = self.parent.join(&format!("{}.rs", ident)); - if adjacent.is_file() { - return self.read_submodule(adjacent); - } - - Err(Error::new( - ErrorKind::NotFound, - format!("No file with module definition for `mod {}`", ident), - )) - } -} - -impl<'a> Folder for Walker<'a> { - fn fold_item(&mut self, mut item: Item) -> Item { - if self.error.is_some() { - return item; // Early return to avoid extra work when erroring - } - - match item.node { - ItemKind::Mod(None) => { - let (attrs, items) = match self.get_attrs_items(&item.attrs, &item.ident) { - Ok(Crate { attrs, items, .. }) => (attrs, items), - Err(e) => { - // Get the file, line, and column information for the - // mod statement we're looking at. - let span = self.fold_span(item.span); - let loc = match self.sm.locinfo(span) { - Ok(li) => li.to_string(), - Err(_) => "unknown location".to_owned(), - }; - - let e = Error::new( - ErrorKind::Other, - ModParseErr { - err: e, - msg: format!( - "Error while parsing `mod {}` \ - statement at {}", - item.ident, - loc - ), - }, - ); - self.error = Some(e); - return item; - } - }; - item.attrs.extend_from_slice(&attrs); - item.node = ItemKind::Mod(Some(items)); - item - } - ItemKind::Mod(Some(items)) => { - let mut parent = self.parent.join(item.ident.as_ref()); - mem::swap(&mut self.parent, &mut parent); - - let items = items.into_iter().map(|item| self.fold_item(item)).collect(); - - mem::swap(&mut self.parent, &mut parent); - item.node = ItemKind::Mod(Some(items)); - item - } - _ => fold::noop_fold_item(self, item), - } - } - - fn fold_span(&mut self, span: Span) -> Span { - let offset = self.sm.files[self.idx].span.lo; - Span { - lo: span.lo + offset, - hi: span.hi + offset, - } - } -} - -/// This is an internal error which is used to build errors when parsing an -/// inner module fails. -#[derive(Debug)] -struct ModParseErr { - err: Error, - msg: String, -} -impl error::Error for ModParseErr { - fn description(&self) -> &str { - &self.msg - } - - fn cause(&self) -> Option<&error::Error> { - Some(&self.err) - } -} -impl fmt::Display for ModParseErr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&self.msg, f)?; - f.write_char('\n')?; - fmt::Display::fmt(&self.err, f) - } -} From 254b784a0b2c375ed2a0223caa9035d827f0b1f7 Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Thu, 30 Aug 2018 21:45:48 +0200 Subject: [PATCH 3/7] Fix nightlies. For some reason, the TokenStream::to_stirng can contains \n when using nightlies --- cpp_common/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp_common/src/lib.rs b/cpp_common/src/lib.rs index ec60f56..93b9fd8 100644 --- a/cpp_common/src/lib.rs +++ b/cpp_common/src/lib.rs @@ -200,7 +200,7 @@ pub mod parsing { cpp: ret.1, // Need to filter the spaces because there is a difference between // proc_macro2 and proc_macro and the hashes would not match - std_body: code.to_string().chars().filter(|x| *x != ' ').collect(), + std_body: code.to_string().chars().filter(|x| *x != ' ' && *x != '\n').collect(), }, body: code.to_string(), callback_offset: 0 From 1acd4e95191e1034bd591fe0cd173d559b61019d Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Thu, 6 Sep 2018 14:42:47 +0200 Subject: [PATCH 4/7] Port to syn 0.15 --- cpp_build/Cargo.toml | 2 +- cpp_build/src/lib.rs | 7 +- cpp_build/src/parser.rs | 111 ++++++++++------- cpp_common/Cargo.toml | 3 +- cpp_common/src/lib.rs | 258 +++++++++++++++++++++------------------- cpp_macros/Cargo.toml | 2 +- cpp_macros/src/lib.rs | 58 ++++++--- 7 files changed, 251 insertions(+), 190 deletions(-) diff --git a/cpp_build/Cargo.toml b/cpp_build/Cargo.toml index cc6a00d..9f18928 100644 --- a/cpp_build/Cargo.toml +++ b/cpp_build/Cargo.toml @@ -14,7 +14,7 @@ documentation = "https://docs.rs/cpp_build" lazy_static = "1.0" cc = "1.0" cpp_common = { path = "../cpp_common", version = "=0.4.0" } -syn = { version = "0.14", features=["full", "visit"] } +syn = { version = "0.15", features=["full", "visit"] } proc-macro2 = "0.4" regex = "1" unicode-xid = "0.1" diff --git a/cpp_build/src/lib.rs b/cpp_build/src/lib.rs index 566dc50..8b16e93 100644 --- a/cpp_build/src/lib.rs +++ b/cpp_build/src/lib.rs @@ -160,9 +160,10 @@ extern "C" {{ let mut sizealign = vec![]; for &Closure { - ref body, + ref body_str, ref sig, ref callback_offset, + .. } in &visitor.closures { let &ClosureSig { @@ -231,7 +232,7 @@ void {name}({params}) {{ ), name = &name, params = params, - body = body + body = body_str ).unwrap(); } else { let comma = if params.is_empty() { "" } else { "," }; @@ -257,7 +258,7 @@ void {name}({params}{comma} void* __result) {{ comma = comma, ty = cpp, args = args, - body = body + body = body_str ).unwrap(); } } diff --git a/cpp_build/src/parser.rs b/cpp_build/src/parser.rs index cf39427..1e35e45 100644 --- a/cpp_build/src/parser.rs +++ b/cpp_build/src/parser.rs @@ -1,4 +1,4 @@ -use cpp_common::{parsing, Class, Closure, Macro}; +use cpp_common::{Class, Closure, Macro, RustInvocation}; use regex::Regex; use std::fmt; use std::fs::File; @@ -15,7 +15,7 @@ pub enum Error { }, ParseSyntaxError { src_path: String, - error: syn::synom::ParseError, + error: syn::parse::Error, }, LexError { src_path: String, @@ -41,6 +41,27 @@ impl fmt::Display for Error { } } +#[derive(Debug)] +struct LineError(u32, String); + +impl LineError { + fn add_line(self, a: u32) -> LineError { + LineError(self.0 + a, self.1) + } +} + +impl fmt::Display for LineError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}:{}", self.0 + 1, self.1) + } +} + +impl From for LineError { + fn from(e: LexError) -> Self { + LineError(e.line, "Lexing error".into()) + } +} + enum ExpandSubMacroType<'a> { Lit, Closure(&'a mut u32), // the offset @@ -49,7 +70,7 @@ enum ExpandSubMacroType<'a> { // Given a string containing some C++ code with a rust! macro, // this functions expand the rust! macro to a call to an extern // function -fn expand_sub_rust_macro(input: String, mut t: ExpandSubMacroType) -> Result { +fn expand_sub_rust_macro(input: String, mut t: ExpandSubMacroType) -> Result { let mut result = input; let mut extra_decl = String::new(); let mut search_index = 0; @@ -59,16 +80,13 @@ fn expand_sub_rust_macro(input: String, mut t: ExpandSubMacroType) -> Result(input) + .map_err(|e| LineError(cursor.line, e.to_string()))?; + (rust_invocation, cursor.off as usize, end.off as usize + 1) }; let fn_name = match t { ExpandSubMacroType::Lit => { @@ -404,10 +422,20 @@ impl Parser { let size = (cursor.off - macro_cur.off) as usize; macro_cur.rest = ¯o_cur.rest[..size]; if ident == "cpp" { - self.handle_cpp(macro_cur)?; + self.handle_cpp(macro_cur).unwrap_or_else(|e| { + panic!( + "Error while parsing cpp! macro:\n{:?}:{}", + self.current_path, e + ) + }); } else { debug_assert_eq!(ident, "cpp_class"); - self.handle_cpp_class(macro_cur)?; + self.handle_cpp_class(macro_cur).unwrap_or_else(|e| { + panic!( + "Error while parsing cpp_class! macro:\n{:?}:{}", + self.current_path, e + ) + }); } continue; } @@ -426,51 +454,44 @@ impl Parser { } } - fn lex_error2(&self) -> Error { - Error::LexError { - src_path: self.current_path.clone().to_str().unwrap().to_owned(), - line: 0, - } - } - - fn handle_cpp(&mut self, x: Cursor) -> Result<(), Error> { + fn handle_cpp(&mut self, x: Cursor) -> Result<(), LineError> { // Since syn don't give the exact string, we extract manually - let begin = (find_delimited(x, "{").map_err(|e| self.lex_error(e))?.0).advance(1); - let end = find_delimited(begin, "}").map_err(|e| self.lex_error(e))?.0; + let begin = (find_delimited(x, "{")?.0).advance(1); + let end = find_delimited(begin, "}")?.0; let extracted = &begin.rest[..(end.off - begin.off) as usize]; - let extracted = line_directive(&self.current_path, begin) + extracted; - let input = syn::buffer::TokenBuffer::new2(x.rest.parse().map_err(|_| self.lex_error2())?); - match parsing::build_macro(input.begin()) - .expect(&format!("cpp! macro at {:?}:{}", self.current_path, x.line)) - .0 - { + let input: ::proc_macro2::TokenStream = x + .rest + .parse() + .map_err(|_| LineError(x.line, "TokenStream parse error".into()))?; + match ::syn::parse2::(input).map_err(|e| LineError(x.line, e.to_string()))? { Macro::Closure(mut c) => { c.callback_offset = self.callbacks_count; - c.body = expand_sub_rust_macro( - extracted.to_string(), - ExpandSubMacroType::Closure(&mut self.callbacks_count), - ).map_err(|e| self.lex_error(e))?; + c.body_str = line_directive(&self.current_path, begin) + + &expand_sub_rust_macro( + extracted.to_string(), + ExpandSubMacroType::Closure(&mut self.callbacks_count), + ).map_err(|e| e.add_line(begin.line))?; self.closures.push(c); } Macro::Lit(_l) => { self.snippets.push('\n'); - let snip = expand_sub_rust_macro(extracted.to_string(), ExpandSubMacroType::Lit) - .map_err(|e| self.lex_error(e))?; + let snip = line_directive(&self.current_path, begin) + + &expand_sub_rust_macro(extracted.to_string(), ExpandSubMacroType::Lit) + .map_err(|e| e.add_line(begin.line))?; self.snippets.push_str(&snip); } } Ok(()) } - fn handle_cpp_class(&mut self, x: Cursor) -> Result<(), Error> { - let input = syn::buffer::TokenBuffer::new2(x.rest.parse().map_err(|_| self.lex_error2())?); - let mut class = parsing::cpp_class(input.begin()) - .expect(&format!( - "cpp_class! macro at {:?}:{}", - self.current_path, x.line - )) - .0; + fn handle_cpp_class(&mut self, x: Cursor) -> Result<(), LineError> { + let input: ::proc_macro2::TokenStream = x + .rest + .parse() + .map_err(|_| LineError(x.line, "TokenStream parse error".into()))?; + let mut class = + ::syn::parse2::(input).map_err(|e| LineError(x.line, e.to_string()))?; class.line = line_directive(&self.current_path, x); self.classes.push(class); Ok(()) diff --git a/cpp_common/Cargo.toml b/cpp_common/Cargo.toml index b4a7862..fe5f05d 100644 --- a/cpp_common/Cargo.toml +++ b/cpp_common/Cargo.toml @@ -11,7 +11,6 @@ repository = "https://github.com/mystor/rust-cpp" documentation = "https://docs.rs/cpp_common" [dependencies] -syn = { version = "0.14", features = ["full", "visit", "extra-traits"] } +syn = { version = "0.15", features = ["full", "extra-traits"] } lazy_static = "1.0" -quote = "0.6" proc-macro2 = "0.4" diff --git a/cpp_common/src/lib.rs b/cpp_common/src/lib.rs index 93b9fd8..72513ba 100644 --- a/cpp_common/src/lib.rs +++ b/cpp_common/src/lib.rs @@ -1,8 +1,6 @@ #[macro_use] extern crate syn; extern crate proc_macro2; -#[macro_use] -extern crate quote; #[macro_use] extern crate lazy_static; @@ -12,7 +10,9 @@ use std::env; use std::hash::{Hash, Hasher}; use std::path::PathBuf; -use proc_macro2::{Span, TokenTree}; +use proc_macro2::{Span, TokenStream, TokenTree}; +use syn::ext::IdentExt; +use syn::parse::{Parse, ParseStream, Result}; use syn::{Attribute, Ident, Type}; pub const VERSION: &'static str = env!("CARGO_PKG_VERSION"); @@ -70,10 +70,24 @@ pub struct Capture { pub cpp: String, } +impl Parse for Capture { + fn parse(input: ParseStream) -> Result { + Ok(Capture { + mutable: input.parse::>()?.is_some(), + // mutable: input.peek(Token![mut]) && { input.parse::()?; true }, + name: input.call(Ident::parse_any)?, + cpp: { + input.parse::()?; + input.parse::()?.value() + }, + }) + } +} + #[derive(Clone, Debug, Hash, PartialEq, Eq)] pub struct ClosureSig { pub captures: Vec, - pub ret: Type, + pub ret: Option, pub cpp: String, pub std_body: String, } @@ -97,10 +111,52 @@ impl ClosureSig { #[derive(Clone, Debug)] pub struct Closure { pub sig: ClosureSig, - pub body: String, + pub body: TokenTree, + pub body_str: String, // with rust! macro replaced pub callback_offset: u32, } +impl Parse for Closure { + fn parse(input: ParseStream) -> Result { + input.parse::>()?; + let cap_con; + bracketed!(cap_con in input); + let captures = syn::punctuated::Punctuated::::parse_terminated( + &cap_con, + )?.into_iter() + .collect(); + let (ret, cpp) = if input.peek(Token![->]) { + input.parse::]>()?; + let t: syn::Type = input.parse()?; + input.parse::()?; + let s = input.parse::()?.value(); + (Some(t), s) + } else { + (None, "void".to_owned()) + }; + let body = input.parse::()?; + // Need to filter the spaces because there is a difference between + // proc_macro2 and proc_macro and the hashes would not match + let std_body = body + .to_string() + .chars() + .filter(|x| *x != ' ' && *x != '\n') + .collect(); + + Ok(Closure { + sig: ClosureSig { + captures, + ret, + cpp, + std_body, + }, + body, + body_str: String::new(), + callback_offset: 0, + }) + } +} + #[derive(Clone, Debug)] pub struct Class { pub name: Ident, @@ -137,10 +193,40 @@ impl Class { } } +impl Parse for Class { + fn parse(input: ParseStream) -> Result { + Ok(Class { + attrs: input.call(Attribute::parse_outer)?, + name: { + input.parse::()?; + input.parse::()?; + input.parse::()?; + input.parse()? + }, + cpp: { + input.parse::()?; + input.parse::()?.value() + }, + line: String::new(), + }) + } +} + #[derive(Debug)] pub enum Macro { Closure(Closure), - Lit(TokenTree), + Lit(TokenStream), +} + +impl Parse for Macro { + fn parse(input: ParseStream) -> Result { + if input.peek(syn::token::Brace) { + let content; + braced!(content in input); + return Ok(Macro::Lit(content.parse()?)); + } + Ok(Macro::Closure(input.parse::()?)) + } } #[derive(Debug)] @@ -152,121 +238,47 @@ pub struct RustInvocation { pub arguments: Vec<(Ident, String)>, // Vec of name and type } -pub mod parsing { - use super::{Capture, Class, Closure, ClosureSig, Macro, RustInvocation}; - use proc_macro2::TokenTree; - use syn::{punctuated::Punctuated, Attribute, Ident, LitStr, Type, Visibility}; - - named!(ident_or_self -> Ident, alt!(syn!(Ident) | - keyword!(self) => { |x| x.into() } )); - - named!(name_as_string -> Capture, - do_parse!( - is_mut: option!(keyword!(mut)) >> - id: ident_or_self >> - keyword!(as) >> - cty: syn!(LitStr) >> - (Capture { - mutable: is_mut.is_some(), - name: id, - cpp: cty.value() - }) - )); - - named!(captures -> Vec, map!(brackets!(call!( - Punctuated::::parse_separated_with, name_as_string)), |x| x.1.into_iter().collect())); - - named!(ret_ty -> (Type, String), - alt!( - do_parse!(punct!(->) >> - rty: syn!(Type) >> - keyword!(as) >> - cty: syn!(LitStr) >> - ((rty, cty.value()))) | - value!((parse_quote!{()}, "void".to_owned())) - )); - - named!(code_block -> TokenTree, syn!(TokenTree)); - - named!(pub cpp_closure -> Closure, - do_parse!(option!(keyword!(unsafe)) >> - captures: captures >> - ret: ret_ty >> - code: code_block >> - (Closure { - sig: ClosureSig { - captures: captures, - ret: ret.0, - cpp: ret.1, - // Need to filter the spaces because there is a difference between - // proc_macro2 and proc_macro and the hashes would not match - std_body: code.to_string().chars().filter(|x| *x != ' ' && *x != '\n').collect(), - }, - body: code.to_string(), - callback_offset: 0 - }))); - - named!(pub build_macro -> Macro , alt!( - cpp_closure => { |c| Macro::Closure(c) } | - code_block => { |b| Macro::Lit(b) } - )); - - named!(pub cpp_class -> Class, - do_parse!( - attrs: many0!(Attribute::parse_outer) >> - option!(syn!(Visibility)) >> - keyword!(unsafe) >> - keyword!(struct) >> - name: syn!(Ident) >> - keyword!(as) >> - cpp_type: syn!(LitStr) >> - (Class { - name: name, - cpp: cpp_type.value(), - attrs: attrs, - line: String::default(), - - }))); - - named!(rust_macro_argument -> (Ident, String), - do_parse!( - name: syn!(Ident) >> - punct!(:) >> - syn!(Type) >> - keyword!(as) >> - cty: syn!(LitStr) >> - ((name, cty.value())))); - - named!(pub find_rust_macro -> RustInvocation, - do_parse!( - begin: custom_keyword!(rust) >> - punct!(!) >> - content: parens!(do_parse!( - id: syn!(Ident) >> - args: map!(brackets!(call!(Punctuated::<(Ident, String), Token![,]>::parse_separated_with, rust_macro_argument)), - |x| x.1.into_iter().collect()) >> - rty : option!( - do_parse!(punct!(->) >> - syn!(Type) >> - keyword!(as) >> - cty: syn!(LitStr) >> - (cty.value()))) >> - syn!(TokenTree) >> - (id, args, rty))) >> - (RustInvocation{ - begin: begin.span(), - end: (content.0).0, - id: (content.1).0, - return_type: (content.1).2, - arguments: (content.1).1 - }))); - - named!(pub find_all_rust_macro -> Vec, - map!(many0!(alt!(find_rust_macro => {|x| vec![x] } - | map!(brackets!(call!(find_all_rust_macro)), |x| x.1) - | map!(parens!(call!(find_all_rust_macro)), |x| x.1) - | map!(braces!(call!(find_all_rust_macro)), |x| x.1) - | syn!(TokenTree) => { |_| vec![] })), - |x| x.into_iter().flat_map(|x|x).collect())); - +impl Parse for RustInvocation { + fn parse(input: ParseStream) -> Result { + let rust_token: Ident = input.parse()?; + if rust_token != "rust" { + return Err(syn::parse::Error::new(rust_token.span(), "expected `rust`")); + } + input.parse::()?; + let content; + let p = parenthesized!(content in input); + let r = RustInvocation { + begin: rust_token.span(), + end: p.span, + id: content.parse()?, + arguments: { + let c2; + bracketed!(c2 in content); + c2.parse_terminated::<(Ident, String), Token![,]>( + |input: ParseStream| -> Result<(Ident, String)> { + let i = input.call(Ident::parse_any)?; + input.parse::()?; + input.parse::()?; + input.parse::()?; + let s = input.parse::()?.value(); + Ok((i, s)) + }, + )? + .into_iter() + .collect() + }, + return_type: if content.peek(Token![->]) { + content.parse::]>()?; + content.parse::()?; + content.parse::()?; + Some(content.parse::()?.value()) + } else { + None + }, + }; + let c3; + braced!(c3 in content); + c3.parse::()?; + Ok(r) + } } diff --git a/cpp_macros/Cargo.toml b/cpp_macros/Cargo.toml index b268113..ad888fb 100644 --- a/cpp_macros/Cargo.toml +++ b/cpp_macros/Cargo.toml @@ -16,7 +16,7 @@ proc-macro = true [dependencies] lazy_static = "1.0" cpp_common = { path = "../cpp_common", version = "=0.4.0" } -syn = { version = "0.14", features=["full", "visit"] } +syn = { version = "0.15", features=["full"] } quote = "0.6" proc-macro2 = "0.4" aho-corasick = "0.6" diff --git a/cpp_macros/src/lib.rs b/cpp_macros/src/lib.rs index fab00a5..42a7164 100644 --- a/cpp_macros/src/lib.rs +++ b/cpp_macros/src/lib.rs @@ -5,6 +5,7 @@ //! documentation](https://docs.rs/cpp). #![recursion_limit = "128"] +#[macro_use] extern crate syn; #[macro_use] @@ -23,7 +24,7 @@ extern crate aho_corasick; extern crate byteorder; -use cpp_common::{flags, parsing, FILE_HASH, LIB_NAME, MSVC_LIB_NAME, OUT_DIR, VERSION}; +use cpp_common::{flags, RustInvocation, FILE_HASH, LIB_NAME, MSVC_LIB_NAME, OUT_DIR, VERSION}; use std::collections::HashMap; use syn::Ident; @@ -124,6 +125,34 @@ fn open_lib_file() -> io::Result { } } +fn find_all_rust_macro( + input: syn::parse::ParseStream, +) -> Result, syn::parse::Error> { + let mut r = Vec::::new(); + while !input.is_empty() { + if input.peek(Ident) { + if let Ok(ri) = input.parse::() { + r.push(ri); + } + } else if input.peek(syn::token::Brace) { + let c; + braced!(c in input); + r.extend(find_all_rust_macro(&c)?); + } else if input.peek(syn::token::Paren) { + let c; + parenthesized!(c in input); + r.extend(find_all_rust_macro(&c)?); + } else if input.peek(syn::token::Bracket) { + let c; + bracketed!(c in input); + r.extend(find_all_rust_macro(&c)?); + } else { + input.parse::()?; + } + } + return Ok(r); +} + /// Strip tokens from the prefix and suffix of the source string, extracting the /// original argument to the cpp! macro. fn macro_text(mut source: &str) -> &str { @@ -176,8 +205,8 @@ pub fn expand_internal(input: proc_macro::TokenStream) -> proc_macro::TokenStrea let source = input.to_string(); let tokens = macro_text(&source); - let input = syn::buffer::TokenBuffer::new(tokens.parse().unwrap()); - let closure = parsing::cpp_closure(input.begin()).expect("cpp! macro").0; + let input: ::proc_macro2::TokenStream = tokens.parse().unwrap(); + let closure = ::syn::parse2::(input).expect("cpp! macro"); // Get the size data compiled by the build macro let size_data = METADATA.get(&closure.sig.name_hash()).expect( @@ -270,7 +299,7 @@ NOTE: They cannot be generated by macro expansion."#, assert!(ret_size == 0, "`void` should have a size of 0!"); quote! { #extern_name(#(#call_args),*); - ::std::mem::transmute::<(), #ret_ty>(()) + ::std::mem::transmute::<(), (#ret_ty)>(()) } } else { quote! { @@ -280,10 +309,10 @@ NOTE: They cannot be generated by macro expansion."#, } }; - let input = syn::buffer::TokenBuffer::new(closure.body.parse().unwrap()); - let rust_invocations = parsing::find_all_rust_macro(input.begin()) - .expect("rust! macro") - .0; + use std::iter::FromIterator; + use syn::parse::Parser; + let input = proc_macro2::TokenStream::from_iter(vec![closure.body].into_iter()); + let rust_invocations = find_all_rust_macro.parse2(input).expect("rust! macro"); let init_callbacks = if !rust_invocations.is_empty() { let rust_cpp_callbacks = Ident::new( &format!("rust_cpp_callbacks{}", *FILE_HASH), @@ -326,17 +355,18 @@ NOTE: They cannot be generated by macro expansion."#, // Perform a compile time check that the sizes match. ::std::mem::forget( ::std::mem::transmute::<_, [u8; #ret_size]>( - ::std::mem::uninitialized::<#ret_ty>())); + ::std::mem::uninitialized::<(#ret_ty)>())); // Perform a runtime check that the sizes match. - assert!(::std::mem::size_of::<#ret_ty>() == #ret_size); - assert!(::std::mem::align_of::<#ret_ty>() == #ret_align); + assert!(::std::mem::size_of::<(#ret_ty)>() == #ret_size); + assert!(::std::mem::align_of::<(#ret_ty)>() == #ret_align); #call } } } }; + result.to_string().parse().unwrap() } @@ -364,10 +394,8 @@ pub fn expand_wrap_class(input: proc_macro::TokenStream) -> proc_macro::TokenStr tokens = &tokens[..tokens.len() - token.len()].trim(); } - let input = syn::buffer::TokenBuffer::new(tokens.parse().unwrap()); - let class = parsing::cpp_class(input.begin()) - .expect("cpp_class! macro") - .0; + let input: ::proc_macro2::TokenStream = tokens.parse().unwrap(); + let class = ::syn::parse2::(input).expect("cpp_class! macro"); let hash = class.name_hash(); From 3536d5c05cb4d4b05459d24df3edecd12426cc6f Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Mon, 10 Sep 2018 09:01:32 +0200 Subject: [PATCH 5/7] Cleanup of syn 0.15 port based on review by mystor Use custom_keyword! for the rust! macro. Better comments and variable names --- cpp_build/src/lib.rs | 4 +-- cpp_build/src/strnom.rs | 6 ++-- cpp_common/src/lib.rs | 79 ++++++++++++++++++++++++----------------- cpp_macros/src/lib.rs | 16 ++++----- 4 files changed, 60 insertions(+), 45 deletions(-) diff --git a/cpp_build/src/lib.rs b/cpp_build/src/lib.rs index 8b16e93..43d11cd 100644 --- a/cpp_build/src/lib.rs +++ b/cpp_build/src/lib.rs @@ -618,12 +618,12 @@ impl Config { // Parse the crate let mut visitor = parser::Parser::default(); - if let Err(x) = visitor.parse_crate(&crate_root) { + if let Err(err) = visitor.parse_crate(&crate_root) { warnln!(r#"-- rust-cpp parse error -- There was an error parsing the crate for the rust-cpp build script: {} In order to provide a better error message, the build script will exit successfully, such that rustc can provide an error message."#, - x + err ); return; } diff --git a/cpp_build/src/strnom.rs b/cpp_build/src/strnom.rs index 23088dc..af88ab2 100644 --- a/cpp_build/src/strnom.rs +++ b/cpp_build/src/strnom.rs @@ -1,4 +1,6 @@ -//! Adapted from [`nom`](https://github.com/Geal/nom). +//! Fork of the equivalent file from the proc-macro2 file. +//! Modified to support line number counting in Cursor. +//! Also contains some function from stable.rs of proc_macro2. #![allow(dead_code)] // Why is this needed ? @@ -164,7 +166,7 @@ fn is_whitespace(ch: char) -> bool { ch.is_whitespace() || ch == '\u{200e}' || ch == '\u{200f}' } -////// +// --- functions from stable.rs #[inline] fn is_ident_start(c: char) -> bool { diff --git a/cpp_common/src/lib.rs b/cpp_common/src/lib.rs index 72513ba..f465404 100644 --- a/cpp_common/src/lib.rs +++ b/cpp_common/src/lib.rs @@ -28,6 +28,11 @@ pub mod flags { pub const IS_TRIVIALLY_DEFAULT_CONSTRUCTIBLE: u32 = 4; } +pub mod kw { + #![allow(non_camel_case_types)] + custom_keyword!(rust); +} + /// This constant is expected to be a unique string within the compiled binary /// which preceeds a definition of the metadata. It begins with /// rustcpp~metadata, which is printable to make it easier to locate when @@ -71,10 +76,11 @@ pub struct Capture { } impl Parse for Capture { + /// Parse a single captured variable inside within a cpp! macro. + /// Example: `mut foo as "int"` fn parse(input: ParseStream) -> Result { Ok(Capture { mutable: input.parse::>()?.is_some(), - // mutable: input.peek(Token![mut]) && { input.parse::()?; true }, name: input.call(Ident::parse_any)?, cpp: { input.parse::()?; @@ -117,14 +123,20 @@ pub struct Closure { } impl Parse for Closure { + /// Parse the inside of a cpp! macro when this macro is a closure. + /// Example: `unsafe [foo as "int"] -> u32 as "int" { /*... */ } fn parse(input: ParseStream) -> Result { input.parse::>()?; - let cap_con; - bracketed!(cap_con in input); + + // Capture + let capture_content; + bracketed!(capture_content in input); let captures = syn::punctuated::Punctuated::::parse_terminated( - &cap_con, + &capture_content, )?.into_iter() .collect(); + + // Optional return type let (ret, cpp) = if input.peek(Token![->]) { input.parse::]>()?; let t: syn::Type = input.parse()?; @@ -134,13 +146,14 @@ impl Parse for Closure { } else { (None, "void".to_owned()) }; + let body = input.parse::()?; // Need to filter the spaces because there is a difference between // proc_macro2 and proc_macro and the hashes would not match let std_body = body .to_string() .chars() - .filter(|x| *x != ' ' && *x != '\n') + .filter(|x| !x.is_whitespace()) .collect(); Ok(Closure { @@ -194,6 +207,8 @@ impl Class { } impl Parse for Class { + /// Parse the inside of a cpp_class! macro. + /// Example: `#[derive(Default)] pub unsafe struct Foobar as "FooBar"` fn parse(input: ParseStream) -> Result { Ok(Class { attrs: input.call(Attribute::parse_outer)?, @@ -219,6 +234,7 @@ pub enum Macro { } impl Parse for Macro { + ///! Parse the inside of a cpp! macro (a literal or a closure) fn parse(input: ParseStream) -> Result { if input.peek(syn::token::Brace) { let content; @@ -239,46 +255,43 @@ pub struct RustInvocation { } impl Parse for RustInvocation { + /// Parse a rust! macro something looking like `rust!(ident [foo : bar as "bar"] { /*...*/ })` fn parse(input: ParseStream) -> Result { - let rust_token: Ident = input.parse()?; - if rust_token != "rust" { - return Err(syn::parse::Error::new(rust_token.span(), "expected `rust`")); - } + let rust_token = input.parse::()?; input.parse::()?; - let content; - let p = parenthesized!(content in input); + let macro_content; + let p = parenthesized!(macro_content in input); let r = RustInvocation { - begin: rust_token.span(), + begin: rust_token.span, end: p.span, - id: content.parse()?, + id: macro_content.parse()?, arguments: { - let c2; - bracketed!(c2 in content); - c2.parse_terminated::<(Ident, String), Token![,]>( - |input: ParseStream| -> Result<(Ident, String)> { - let i = input.call(Ident::parse_any)?; - input.parse::()?; - input.parse::()?; - input.parse::()?; - let s = input.parse::()?.value(); - Ok((i, s)) - }, - )? + let capture_content; + bracketed!(capture_content in macro_content); + capture_content + .parse_terminated::<(Ident, String), Token![,]>( + |input: ParseStream| -> Result<(Ident, String)> { + let i = input.call(Ident::parse_any)?; + input.parse::()?; + input.parse::()?; + input.parse::()?; + let s = input.parse::()?.value(); + Ok((i, s)) + }, + )? .into_iter() .collect() }, - return_type: if content.peek(Token![->]) { - content.parse::]>()?; - content.parse::()?; - content.parse::()?; - Some(content.parse::()?.value()) + return_type: if macro_content.peek(Token![->]) { + macro_content.parse::]>()?; + macro_content.parse::()?; + macro_content.parse::()?; + Some(macro_content.parse::()?.value()) } else { None }, }; - let c3; - braced!(c3 in content); - c3.parse::()?; + macro_content.parse::()?; Ok(r) } } diff --git a/cpp_macros/src/lib.rs b/cpp_macros/src/lib.rs index 42a7164..25a582d 100644 --- a/cpp_macros/src/lib.rs +++ b/cpp_macros/src/lib.rs @@ -24,8 +24,10 @@ extern crate aho_corasick; extern crate byteorder; -use cpp_common::{flags, RustInvocation, FILE_HASH, LIB_NAME, MSVC_LIB_NAME, OUT_DIR, VERSION}; +use cpp_common::{flags, kw, RustInvocation, FILE_HASH, LIB_NAME, MSVC_LIB_NAME, OUT_DIR, VERSION}; use std::collections::HashMap; +use std::iter::FromIterator; +use syn::parse::Parser; use syn::Ident; use aho_corasick::{AcAutomaton, Automaton}; @@ -130,7 +132,7 @@ fn find_all_rust_macro( ) -> Result, syn::parse::Error> { let mut r = Vec::::new(); while !input.is_empty() { - if input.peek(Ident) { + if input.peek(kw::rust) { if let Ok(ri) = input.parse::() { r.push(ri); } @@ -309,9 +311,7 @@ NOTE: They cannot be generated by macro expansion."#, } }; - use std::iter::FromIterator; - use syn::parse::Parser; - let input = proc_macro2::TokenStream::from_iter(vec![closure.body].into_iter()); + let input = proc_macro2::TokenStream::from_iter([closure.body].iter().map(|x| x.clone())); let rust_invocations = find_all_rust_macro.parse2(input).expect("rust! macro"); let init_callbacks = if !rust_invocations.is_empty() { let rust_cpp_callbacks = Ident::new( @@ -381,7 +381,7 @@ pub fn expand_wrap_class(input: proc_macro::TokenStream) -> proc_macro::TokenStr let s = source .find("stringify!(") - .expect("expected 'strignify!' token in class content") + 11; + .expect("expected 'stringify!' token in class content") + 11; let mut tokens: &str = &source[s..].trim(); for token in SUFFIX.iter().rev() { @@ -394,8 +394,8 @@ pub fn expand_wrap_class(input: proc_macro::TokenStream) -> proc_macro::TokenStr tokens = &tokens[..tokens.len() - token.len()].trim(); } - let input: ::proc_macro2::TokenStream = tokens.parse().unwrap(); - let class = ::syn::parse2::(input).expect("cpp_class! macro"); + let input: proc_macro2::TokenStream = tokens.parse().unwrap(); + let class = syn::parse2::(input).expect("cpp_class! macro"); let hash = class.name_hash(); From cdc63845086bc7a4ff96ad82477c4185961fd69e Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Mon, 10 Sep 2018 09:30:03 +0200 Subject: [PATCH 6/7] Do not go through the string representation of the token So keep the original span --- cpp_macros/src/lib.rs | 166 ++++++++++++++++++------------------------ 1 file changed, 72 insertions(+), 94 deletions(-) diff --git a/cpp_macros/src/lib.rs b/cpp_macros/src/lib.rs index 25a582d..829a956 100644 --- a/cpp_macros/src/lib.rs +++ b/cpp_macros/src/lib.rs @@ -155,44 +155,14 @@ fn find_all_rust_macro( return Ok(r); } -/// Strip tokens from the prefix and suffix of the source string, extracting the -/// original argument to the cpp! macro. -fn macro_text(mut source: &str) -> &str { - #[cfg_attr(rustfmt, rustfmt_skip)] - const PREFIX: &'static [&'static str] = &[ - "#", "[", "allow", "(", "unused", ")", "]", - "enum", "CppClosureInput", "{", - "Input", "=", "(", "stringify", "!", "(" - ]; - - #[cfg_attr(rustfmt, rustfmt_skip)] - const SUFFIX: &'static [&'static str] = &[ - ")", ",", "0", ")", ".", "1", ",", "}" - ]; - - source = source.trim(); - - for token in PREFIX { - assert!( - source.starts_with(token), - "expected prefix token {}, got {}", - token, - source - ); - source = &source[token.len()..].trim(); - } - - for token in SUFFIX.iter().rev() { - assert!( - source.ends_with(token), - "expected suffix token {}, got {}", - token, - source - ); - source = &source[..source.len() - token.len()].trim(); - } - - source +macro_rules! unwrap_enum { + ($e:expr => $what:path) => { + if let $what(ref x) = $e { + x + } else { + panic!("unexpected expression") + } + }; } #[proc_macro_derive(__cpp_internal_closure)] @@ -204,29 +174,46 @@ pub fn expand_internal(input: proc_macro::TokenStream) -> proc_macro::TokenStrea ); // Parse the macro input - let source = input.to_string(); - let tokens = macro_text(&source); - - let input: ::proc_macro2::TokenStream = tokens.parse().unwrap(); - let closure = ::syn::parse2::(input).expect("cpp! macro"); + let item_enum = parse_macro_input!(input as syn::ItemEnum); + let expr = &item_enum + .variants + .first() + .unwrap() + .value() + .clone() + .discriminant + .as_ref() + .unwrap() + .1; + let expr = unwrap_enum!(expr => syn::Expr::Field); + let expr = unwrap_enum!(*expr.base => syn::Expr::Tuple); + let expr = unwrap_enum!(expr.elems.first().unwrap().value().clone() => syn::Expr::Macro); + + let closure = match ::syn::parse2::(expr.mac.tts.clone()) { + Ok(x) => x, + Err(err) => return err.to_compile_error().into(), + }; // Get the size data compiled by the build macro - let size_data = METADATA.get(&closure.sig.name_hash()).expect( - r#" --- rust-cpp fatal error -- - -This cpp! macro is not found in the library's rust-cpp metadata. + let size_data = match METADATA.get(&closure.sig.name_hash()) { + Some(x) => x, + None => { + return quote!(compile_error!{ +r#"This cpp! macro is not found in the library's rust-cpp metadata. NOTE: Only cpp! macros found directly in the program source will be parsed - -NOTE: They cannot be generated by macro expansion."#, - ); +NOTE: They cannot be generated by macro expansion."#}) + .into() + } + }; let mut extern_params = Vec::new(); let mut tt_args = Vec::new(); let mut call_args = Vec::new(); for (i, capture) in closure.sig.captures.iter().enumerate() { let written_name = &capture.name; - let mac_name = Ident::new(&format!("var_{}", written_name), Span::call_site()); - let mac_cty = Ident::new(&format!("cty_{}", written_name), Span::call_site()); + let span = written_name.span(); + let mac_name = Ident::new(&format!("var_{}", written_name), span); + let mac_cty = Ident::new(&format!("cty_{}", written_name), span); // Generate the assertion to check that the size and align of the types // match before calling. @@ -241,7 +228,7 @@ NOTE: They cannot be generated by macro expansion."#, rust", &capture.name ); - let assertion = quote!{ + let assertion = quote_spanned!{span=> // Perform a compile time check that the sizes match. This should be // a no-op. ::std::mem::forget( @@ -256,23 +243,23 @@ NOTE: They cannot be generated by macro expansion."#, }; let mb_mut = if capture.mutable { - quote!(mut) + quote_spanned!(span=> mut) } else { quote!() }; let ptr = if capture.mutable { - quote!(*mut) + quote_spanned!(span=> *mut) } else { - quote!(*const) + quote_spanned!(span=> *const) }; - let arg_name = Ident::new(&format!("arg_{}", written_name), Span::call_site()); + let arg_name = Ident::new(&format!("arg_{}", written_name), span); - extern_params.push(quote!(#arg_name : #ptr u8)); + extern_params.push(quote_spanned!(span=> #arg_name : #ptr u8)); - tt_args.push(quote!(#mb_mut $#mac_name : ident as $#mac_cty : tt)); + tt_args.push(quote_spanned!(span=> #mb_mut $#mac_name : ident as $#mac_cty : tt)); - call_args.push(quote!({ + call_args.push(quote_spanned!(span=> { #assertion &#mb_mut $#mac_name as #ptr _ as #ptr u8 })); @@ -304,7 +291,7 @@ NOTE: They cannot be generated by macro expansion."#, ::std::mem::transmute::<(), (#ret_ty)>(()) } } else { - quote! { + quote!{ let mut result: #ret_ty = ::std::mem::uninitialized(); #extern_name(#(#call_args,)* &mut result); result @@ -367,46 +354,37 @@ NOTE: They cannot be generated by macro expansion."#, } }; - result.to_string().parse().unwrap() + result.into() } #[proc_macro_derive(__cpp_internal_class)] pub fn expand_wrap_class(input: proc_macro::TokenStream) -> proc_macro::TokenStream { - let source = input.to_string(); - - #[cfg_attr(rustfmt, rustfmt_skip)] - const SUFFIX: &'static [&'static str] = &[ - ")", ",", "0", ")", ".", "1", "]", ",", "}" - ]; - - let s = source - .find("stringify!(") - .expect("expected 'stringify!' token in class content") + 11; - let mut tokens: &str = &source[s..].trim(); - - for token in SUFFIX.iter().rev() { - assert!( - tokens.ends_with(token), - "expected suffix token {}, got {}", - token, - tokens - ); - tokens = &tokens[..tokens.len() - token.len()].trim(); - } - - let input: proc_macro2::TokenStream = tokens.parse().unwrap(); - let class = syn::parse2::(input).expect("cpp_class! macro"); + // Parse the macro input + let item_struct = parse_macro_input!(input as syn::ItemStruct); + let expr = unwrap_enum!(&item_struct.fields.iter().nth(0).unwrap().ty => syn::Type::Array); + let expr = unwrap_enum!(expr.len => syn::Expr::Binary); + let expr = unwrap_enum!(*expr.right => syn::Expr::Field); + let expr = unwrap_enum!(*expr.base => syn::Expr::Tuple); + let expr = unwrap_enum!(expr.elems.first().unwrap().value().clone() => syn::Expr::Macro); + + let class = match ::syn::parse2::(expr.mac.tts.clone()) { + Ok(x) => x, + Err(err) => return err.to_compile_error().into(), + }; let hash = class.name_hash(); - let size_data = METADATA.get(&hash).expect( - r#" --- rust-cpp fatal error -- - -This cpp_class! macro is not found in the library's rust-cpp metadata. + // Get the size data compiled by the build macro + let size_data = match METADATA.get(&hash) { + Some(x) => x, + None => { + return quote!(compile_error!{ +r#"This cpp_class! macro is not found in the library's rust-cpp metadata. NOTE: Only cpp_class! macros found directly in the program source will be parsed - -NOTE: They cannot be generated by macro expansion."#, - ); +NOTE: They cannot be generated by macro expansion."#}) + .into() + } + }; let (size, align) = (size_data[0].size, size_data[0].align); @@ -561,5 +539,5 @@ NOTE: They cannot be generated by macro expansion."#, panic!("Deriving from Debug is not implemented") }; - result.to_string().parse().unwrap() + result.into() } From 0a75608e3e71c16bb7149de813454901fc12b778 Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Mon, 10 Sep 2018 12:23:31 +0200 Subject: [PATCH 7/7] Use syn::visit to extract the strignify macro instead of destructuring --- cpp_macros/Cargo.toml | 2 +- cpp_macros/src/lib.rs | 50 ++++++++++++++++--------------------------- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/cpp_macros/Cargo.toml b/cpp_macros/Cargo.toml index ad888fb..2f00377 100644 --- a/cpp_macros/Cargo.toml +++ b/cpp_macros/Cargo.toml @@ -16,7 +16,7 @@ proc-macro = true [dependencies] lazy_static = "1.0" cpp_common = { path = "../cpp_common", version = "=0.4.0" } -syn = { version = "0.15", features=["full"] } +syn = { version = "0.15", features=["full", "visit"] } quote = "0.6" proc-macro2 = "0.4" aho-corasick = "0.6" diff --git a/cpp_macros/src/lib.rs b/cpp_macros/src/lib.rs index 829a956..7ef2787 100644 --- a/cpp_macros/src/lib.rs +++ b/cpp_macros/src/lib.rs @@ -155,14 +155,20 @@ fn find_all_rust_macro( return Ok(r); } -macro_rules! unwrap_enum { - ($e:expr => $what:path) => { - if let $what(ref x) = $e { - x - } else { - panic!("unexpected expression") +/// Find the occurence of the strignify! macro within the macro derive +fn extract_original_macro(input: &syn::DeriveInput) -> Option { + #[derive(Default)] + struct Finder(Option); + impl<'ast> syn::visit::Visit<'ast> for Finder { + fn visit_macro(&mut self, mac: &'ast syn::Macro) { + if mac.path.segments.len() == 1 && mac.path.segments[0].ident == "stringify" { + self.0 = Some(mac.tts.clone()); + } } - }; + } + let mut f = Finder::default(); + syn::visit::visit_derive_input(&mut f, &input); + f.0 } #[proc_macro_derive(__cpp_internal_closure)] @@ -174,22 +180,9 @@ pub fn expand_internal(input: proc_macro::TokenStream) -> proc_macro::TokenStrea ); // Parse the macro input - let item_enum = parse_macro_input!(input as syn::ItemEnum); - let expr = &item_enum - .variants - .first() - .unwrap() - .value() - .clone() - .discriminant - .as_ref() - .unwrap() - .1; - let expr = unwrap_enum!(expr => syn::Expr::Field); - let expr = unwrap_enum!(*expr.base => syn::Expr::Tuple); - let expr = unwrap_enum!(expr.elems.first().unwrap().value().clone() => syn::Expr::Macro); - - let closure = match ::syn::parse2::(expr.mac.tts.clone()) { + let input = extract_original_macro(&parse_macro_input!(input as syn::DeriveInput)).unwrap(); + + let closure = match syn::parse2::(input) { Ok(x) => x, Err(err) => return err.to_compile_error().into(), }; @@ -360,14 +353,9 @@ NOTE: They cannot be generated by macro expansion."#}) #[proc_macro_derive(__cpp_internal_class)] pub fn expand_wrap_class(input: proc_macro::TokenStream) -> proc_macro::TokenStream { // Parse the macro input - let item_struct = parse_macro_input!(input as syn::ItemStruct); - let expr = unwrap_enum!(&item_struct.fields.iter().nth(0).unwrap().ty => syn::Type::Array); - let expr = unwrap_enum!(expr.len => syn::Expr::Binary); - let expr = unwrap_enum!(*expr.right => syn::Expr::Field); - let expr = unwrap_enum!(*expr.base => syn::Expr::Tuple); - let expr = unwrap_enum!(expr.elems.first().unwrap().value().clone() => syn::Expr::Macro); - - let class = match ::syn::parse2::(expr.mac.tts.clone()) { + let input = extract_original_macro(&parse_macro_input!(input as syn::DeriveInput)).unwrap(); + + let class = match ::syn::parse2::(input) { Ok(x) => x, Err(err) => return err.to_compile_error().into(), };