[WIP] add saltwater backend

Cargo.toml

@@ -46,7 +46,6 @@ shlex = "0.1"
 
 [dependencies]
 bitflags = "1.0.3"
-cexpr = "0.4"
 cfg-if = "0.1.0"
 # This kinda sucks: https://github.com/rust-lang/cargo/issues/1982
 clap = { version = "2", optional = true }
@@ -59,6 +58,8 @@ regex = { version = "1.0", default-features = false , features = [ "std", "unico
 which = { version = "3.0", optional = true, default-features = false }
 shlex = "0.1"
 rustc-hash = "1.0.1"
+saltwater = { version = "0.10", default-features = false }
+
 # New validation in 0.3.6 breaks bindgen-integration:
 # https://github.com/alexcrichton/proc-macro2/commit/489c642.
 proc-macro2 = { version = "1", default-features = false }

src/clang.rs

@@ -4,9 +4,7 @@
 #![allow(non_upper_case_globals, dead_code)]
 
 use crate::ir::context::BindgenContext;
-use cexpr;
 use clang_sys::*;
-use regex;
 use std::ffi::{CStr, CString};
 use std::fmt;
 use std::hash::Hash;
@@ -696,11 +694,11 @@ impl Cursor {
         RawTokens::new(self)
     }
 
-    /// Gets the tokens that correspond to that cursor as  `cexpr` tokens.
-    pub fn cexpr_tokens(self) -> Vec<cexpr::token::Token> {
+    /// Gets the tokens that correspond to that cursor as `saltwater` tokens.
+    pub fn swcc_tokens(self) -> Vec<saltwater::Locatable<saltwater::Token>> {
         self.tokens()
             .iter()
-            .filter_map(|token| token.as_cexpr_token())
+            .filter_map(|token| token.as_swcc_token())
             .collect()
     }
 
@@ -795,28 +793,69 @@ impl ClangToken {
         c_str.to_bytes()
     }
 
-    /// Converts a ClangToken to a `cexpr` token if possible.
-    pub fn as_cexpr_token(&self) -> Option<cexpr::token::Token> {
-        use cexpr::token;
+    /// Converts a ClangToken to an `saltwater` token if possible.
+    pub fn as_swcc_token(
+        &self,
+    ) -> Option<saltwater::Locatable<saltwater::Token>> {
+        use saltwater::{
+            error::LexError, Files, Lexer, Literal, Locatable, Token,
+        };
 
-        let kind = match self.kind {
-            CXToken_Punctuation => token::Kind::Punctuation,
-            CXToken_Literal => token::Kind::Literal,
-            CXToken_Identifier => token::Kind::Identifier,
-            CXToken_Keyword => token::Kind::Keyword,
-            // NB: cexpr is not too happy about comments inside
-            // expressions, so we strip them down here.
+        match self.kind {
+            // `saltwater` does not have a comment token
             CXToken_Comment => return None,
+            CXToken_Punctuation | CXToken_Literal | CXToken_Identifier |
+            CXToken_Keyword => {
+                let spelling = std::str::from_utf8(self.spelling())
+                    .expect("invalid utf8 in token");
+                let parse = |spelling| {
+                    let mut files = Files::new();
+                    let id = files.add("", "".into());
+                    let mut lexer = Lexer::new(id, spelling, false);
+                    lexer.next().unwrap()
+                };
+                let failed_parse = |err: Locatable<_>| {
+                    panic!(
+                        "saltwater failed to parse clang token '{}': {}",
+                        spelling, err.data
+                    );
+                };
+                let mut token = match parse(spelling) {
+                    Ok(token) => token,
+                    Err(Locatable {
+                        data:
+                            LexError::IntegerOverflow {
+                                is_signed: Some(true),
+                            },
+                        ..
+                    }) => {
+                        warn!("integer does not fit into `long long`, trying again with `unsigned long long`");
+                        // saltwater ignores trailing `LL`, but requires any `u` suffix to come before `LL`
+                        let mut spelling = String::from(
+                            spelling
+                                .trim_end_matches('l')
+                                .trim_end_matches('L'),
+                        );
+                        spelling.push('u');
+                        parse(&spelling).unwrap_or_else(failed_parse)
+                    }
+                    Err(err) => failed_parse(err),
+                };
+
+                // saltwater generates null-terminated string immediately,
+                // but bindgen only adds the null-terminator during codegen.
+                if let Token::Literal(Literal::Str(ref mut string)) =
+                    &mut token.data
+                {
+                    assert_eq!(string.pop(), Some(b'\0'));
+                }
+                Some(token)
+            }
             _ => {
                 warn!("Found unexpected token kind: {:?}", self);
-                return None;
+                None
             }
-        };
-
-        Some(token::Token {
-            kind,
-            raw: self.spelling().to_vec().into_boxed_slice(),
-        })
+        }
     }
 }
 
@@ -836,11 +875,11 @@ impl<'a> Iterator for ClangTokenIterator<'a> {
     type Item = ClangToken;
 
     fn next(&mut self) -> Option<Self::Item> {
-        let raw = self.raw.next()?;
+        let raw = *self.raw.next()?;
         unsafe {
-            let kind = clang_getTokenKind(*raw);
-            let spelling = clang_getTokenSpelling(self.tu, *raw);
-            let extent = clang_getTokenExtent(self.tu, *raw);
+            let kind = clang_getTokenKind(raw);
+            let spelling = clang_getTokenSpelling(self.tu, raw);
+            let extent = clang_getTokenExtent(self.tu, raw);
             Some(ClangToken {
                 kind,
                 extent,