Skip to content

Commit

Permalink
lang: Migrate to nom 4.0.0
Browse files Browse the repository at this point in the history
As a workaround to rust-bakery/nom#544, migrate to
nom 4 to ensure that the verbose-errors feature becomes additive and
therefore portus compiles when used as a dependency.

There were two classes of changes to the parser structure:
1. Error handling, as predicted and outlined here:
https://github.com/Geal/nom/blob/master/doc/upgrading_to_nom_4.md#replacing-parser-result-matchers

2. Migration to "CompleteByteSlice".
This was predicted in the migration notes:
https://github.com/Geal/nom/blob/master/doc/upgrading_to_nom_4.md#dealing-with-incomplete-usage
but caused underlying errors as reported here:
rust-bakery/nom#790 (comment)

To address this, shadow nom's `named!` macro with our own,
`named_complete!`, which replaces the types appropriately. This is the
solution proposed here:
rust-bakery/nom#795 (comment)
  • Loading branch information
akshayknarayan committed Sep 6, 2018
1 parent b1cd757 commit 16eba9b
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 103 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ bytes = "0.4.5"
clap = "2.29"
libc = "0.2"
nix = "0.9.0"
nom = "^3.2"
nom = "^4"
slog = "2"
slog-async = "2"
slog-term = "2"
Expand Down
106 changes: 68 additions & 38 deletions src/lang/ast.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use nom::IResult;
use nom;
use super::{Error, Result};

#[derive(Clone, Debug, PartialEq)]
Expand Down Expand Up @@ -51,7 +51,7 @@ pub enum Expr {
}

use std::str;
named!(
named_complete!(
op<Result<Op>>,
alt!(
alt!(tag!("+") | tag!("add")) => { |_| Ok(Op::Add) } |
Expand Down Expand Up @@ -90,7 +90,7 @@ fn check_expr(op: Op, left: Expr, right: Expr) -> Result<Expr> {
}

use nom::multispace;
named!(
named_complete!(
sexp<Result<Expr>>,
ws!(delimited!(
tag!("("),
Expand All @@ -111,26 +111,25 @@ named!(
);

use nom::digit;
use nom::types::CompleteByteSlice;
use std::str::FromStr;
named!(
named_complete!(
pub num<u64>,
map_res!(
map_res!(digit, str::from_utf8),
FromStr::from_str
digit,
|d: CompleteByteSlice| {
let st = str::from_utf8(d.0)?;
FromStr::from_str(st).map_err(Error::from)
}
)
);

use nom::is_alphanumeric;
named!(
name_raw<&[u8]>,
take_while1!(|u: u8| is_alphanumeric(u) || u == b'.' || u == b'_')
);

named!(
named_complete!(
pub name<String>,
map_res!(
name_raw,
|n: &[u8]| str::from_utf8(n).map_err(Error::from).and_then(|s|
take_while1!(|u: u8| is_alphanumeric(u) || u == b'.' || u == b'_'),
|n: CompleteByteSlice| str::from_utf8(n.0).map_err(Error::from).and_then(|s|
if s.starts_with("__") {
Err(Error::from(
format!("Names beginning with \"__\" are reserved for internal use: {:?}", s),
Expand All @@ -142,7 +141,7 @@ named!(
)
);

named!(
named_complete!(
pub atom<Result<Expr>>,
ws!(do_parse!(
val: alt!(
Expand All @@ -156,7 +155,7 @@ named!(
))
);

named!(
named_complete!(
command<Result<Expr>>,
ws!(delimited!(
tag!("("),
Expand All @@ -171,7 +170,7 @@ named!(
))
);

named!(
named_complete!(
pub comment<Result<Expr>>,
ws!(do_parse!(
tag!("#") >>
Expand All @@ -180,12 +179,12 @@ named!(
))
);

named!(
named_complete!(
pub expr<Result<Expr>>,
alt_complete!(comment | sexp | command | atom)
);

named!(
named_complete!(
pub exprs<Vec<Result<Expr>>>,
many1!(expr)
);
Expand All @@ -194,14 +193,15 @@ impl Expr {
// TODO make return Iter
pub fn new(src: &[u8]) -> Result<Vec<Self>> {
use nom::Needed;
match exprs(src) {
IResult::Done(_, me) => me.into_iter().filter(|e| match e {
match exprs(CompleteByteSlice(src)) {
Ok((_, me)) => me.into_iter().filter(|e| match e {
Ok(Expr::None) => false,
_ => true,
}).collect(),
IResult::Error(e) => Err(Error::from(e)),
IResult::Incomplete(Needed::Unknown) => Err(Error::from("need more src")),
IResult::Incomplete(Needed::Size(s)) => Err(
Err(nom::Err::Error(e)) |
Err(nom::Err::Failure(e)) => Err(Error::from(e)),
Err(nom::Err::Incomplete(Needed::Unknown)) => Err(Error::from("need more src")),
Err(nom::Err::Incomplete(Needed::Size(s))) => Err(
Error::from(format!("need {} more bytes", s)),
),
}
Expand Down Expand Up @@ -235,47 +235,77 @@ impl Expr {

#[cfg(test)]
mod tests {
use nom::types::CompleteByteSlice;
use super::{Command, Expr, Op, Prim};

#[test]
fn atom() {
fn atom_0() {
use super::name;
let foo = b"foo";
let er = name(CompleteByteSlice(foo));
println!("{:?}", er.expect("parse single atom"));
}

#[test]
fn atom_1() {
let foo = b"1";
let er = Expr::new(foo);
let e = er.unwrap();
assert_eq!(e, vec![Expr::Atom(Prim::Num(1))]);

}

#[test]
fn atom_2() {
let foo = b"1 ";
let er = Expr::new(foo);
let e = er.unwrap();
assert_eq!(e, vec![Expr::Atom(Prim::Num(1))]);
}

#[test]
fn atom_3() {
let foo = b"+";
let er = Expr::new(foo);
match er {
Ok(e) => panic!("false ok: {:?}", e),
Err(_) => (),
}
}

#[test]
fn atom_4() {
let foo = b"true";
let er = Expr::new(foo);
let e = er.unwrap();
assert_eq!(e, vec![Expr::Atom(Prim::Bool(true))]);
}

#[test]
fn atom_5() {
let foo = b"false";
let er = Expr::new(foo);
let e = er.unwrap();
assert_eq!(e, vec![Expr::Atom(Prim::Bool(false))]);
}

#[test]
fn atom_6() {
let foo = b"x";
let er = Expr::new(foo);
let e = er.unwrap();
assert_eq!(e, vec![Expr::Atom(Prim::Name(String::from("x")))]);
}

#[test]
fn atom_7() {
let foo = b"acbdefg";
let er = Expr::new(foo);
let e = er.unwrap();
assert_eq!(e, vec![Expr::Atom(Prim::Name(String::from("acbdefg")))]);

}

#[test]
fn atom_8() {
let foo = b"blah 10 20";
let er = Expr::new(foo);
let e = er.unwrap();
Expand All @@ -288,7 +318,7 @@ mod tests {
]
);
}

#[test]
fn simple_exprs() {
let foo = b"(+ 10 20)";
Expand Down Expand Up @@ -353,13 +383,14 @@ mod tests {

#[test]
fn expr_leftover() {
use nom;
let foo = b"(+ 10 20))";
use nom::{IResult, Needed};
use nom::Needed;
use super::exprs;
use lang::{Error, Result};
match exprs(foo) {
IResult::Done(r, me) => {
assert_eq!(r, b")");
use lang::Result;
match exprs(CompleteByteSlice(foo)) {
Ok((r, me)) => {
assert_eq!(r, CompleteByteSlice(b")"));
assert_eq!(
me.into_iter().collect::<Result<Vec<Expr>>>().unwrap(),
vec![
Expand All @@ -371,11 +402,10 @@ mod tests {
],
);
},
IResult::Error(e) => panic!(e),
IResult::Incomplete(Needed::Unknown) => panic!("need more src"),
IResult::Incomplete(Needed::Size(s)) => panic!(
Error::from(format!("need {} more bytes", s)),
),
Err(nom::Err::Error(e)) |
Err(nom::Err::Failure(e)) => panic!(e),
Err(nom::Err::Incomplete(Needed::Unknown)) => panic!("incomplete"),
Err(nom::Err::Incomplete(Needed::Size(s))) => panic!("need {} more bytes", s),
}
}

Expand Down
35 changes: 30 additions & 5 deletions src/lang/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -130,22 +130,47 @@ impl<'a> From<&'a str> for Error {
Error(String::from(e))
}
}
impl From<nom::simple_errors::Err> for Error {
fn from(e: nom::simple_errors::Err) -> Error {
Error(String::from(e.description()))
impl<I, E> From<nom::Err<I, E>> for Error {
fn from(e: nom::Err<I, E>) -> Error {
Error(String::from(e.into_error_kind().description()))
}
}
impl<I, E> From<nom::Context<I, E>> for Error {
fn from(e: nom::Context<I, E>) -> Error {
Error(String::from(e.into_error_kind().description()))
}
}
impl From<std::string::FromUtf8Error> for Error {
fn from(e: std::string::FromUtf8Error) -> Error {
Error(format!("err {}", e))
Error(format!("string err {}", e))
}
}
impl From<std::str::Utf8Error> for Error {
fn from(e: std::str::Utf8Error) -> Error {
Error(format!("err {}", e))
Error(format!("string err {}", e))
}
}
impl From<std::num::ParseIntError> for Error {
fn from(e: std::num::ParseIntError) -> Error {
Error(format!("int err {}", e))
}
}

/// Define this helper macro to replace the named! macro provided by nom
/// to address https://github.com/Geal/nom/issues/790 with CompleteByteSlice
macro_rules! named_complete {
($name:ident<$t:ty>, $submac:ident!( $($args:tt)* )) => (
fn $name( i: nom::types::CompleteByteSlice ) -> nom::IResult<nom::types::CompleteByteSlice, $t, u32> {
$submac!(i, $($args)*)
}
);
(pub $name:ident<$t:ty>, $submac:ident!( $($args:tt)* )) => (
pub fn $name( i: nom::types::CompleteByteSlice ) -> nom::IResult<nom::types::CompleteByteSlice, $t, u32> {
$submac!(i, $($args)*)
}
)
}

mod ast;
mod datapath;
mod prog;
Expand Down
Loading

0 comments on commit 16eba9b

Please sign in to comment.