Skip to content

Commit

Permalink
Merge pull request #275 from epage/p2
Browse files Browse the repository at this point in the history
feat(parse): Allow mutable parsing
  • Loading branch information
epage authored Jul 6, 2023
2 parents 0625541 + 77ef941 commit c4d3cee
Show file tree
Hide file tree
Showing 56 changed files with 980 additions and 948 deletions.
20 changes: 10 additions & 10 deletions benches/contains_token.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,19 @@ fn contains_token(c: &mut criterion::Criterion) {
group.throughput(criterion::Throughput::Bytes(len as u64));

group.bench_with_input(criterion::BenchmarkId::new("slice", name), &len, |b, _| {
b.iter(|| black_box(parser_slice.parse_next(black_box(sample)).unwrap()));
b.iter(|| black_box(parser_slice.parse_peek(black_box(sample)).unwrap()));
});
group.bench_with_input(criterion::BenchmarkId::new("array", name), &len, |b, _| {
b.iter(|| black_box(parser_array.parse_next(black_box(sample)).unwrap()));
b.iter(|| black_box(parser_array.parse_peek(black_box(sample)).unwrap()));
});
group.bench_with_input(criterion::BenchmarkId::new("tuple", name), &len, |b, _| {
b.iter(|| black_box(parser_tuple.parse_next(black_box(sample)).unwrap()));
b.iter(|| black_box(parser_tuple.parse_peek(black_box(sample)).unwrap()));
});
group.bench_with_input(
criterion::BenchmarkId::new("closure-or", name),
&len,
|b, _| {
b.iter(|| black_box(parser_closure_or.parse_next(black_box(sample)).unwrap()));
b.iter(|| black_box(parser_closure_or.parse_peek(black_box(sample)).unwrap()));
},
);
group.bench_with_input(
Expand All @@ -40,7 +40,7 @@ fn contains_token(c: &mut criterion::Criterion) {
b.iter(|| {
black_box(
parser_closure_matches
.parse_next(black_box(sample))
.parse_peek(black_box(sample))
.unwrap(),
)
});
Expand All @@ -52,17 +52,17 @@ fn contains_token(c: &mut criterion::Criterion) {

fn parser_slice(input: &str) -> IResult<&str, usize> {
let contains = &['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'][..];
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_next(input)
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_peek(input)
}

fn parser_array(input: &str) -> IResult<&str, usize> {
let contains = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'];
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_next(input)
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_peek(input)
}

fn parser_tuple(input: &str) -> IResult<&str, usize> {
let contains = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9');
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_next(input)
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_peek(input)
}

fn parser_closure_or(input: &str) -> IResult<&str, usize> {
Expand All @@ -78,12 +78,12 @@ fn parser_closure_or(input: &str) -> IResult<&str, usize> {
|| c == '8'
|| c == '9'
};
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_next(input)
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_peek(input)
}

fn parser_closure_matches(input: &str) -> IResult<&str, usize> {
let contains = |c: char| matches!(c, '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9');
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_next(input)
repeat(0.., alt((take_while(1.., contains), take_till1(contains)))).parse_peek(input)
}

const CONTIGUOUS: &str = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
Expand Down
8 changes: 4 additions & 4 deletions examples/arithmetic/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ pub fn expr(i: &str) -> IResult<&str, i64> {
}
},
)
.parse_next(i)
.parse_peek(i)
}

// We read an initial factor and for each time we find
Expand All @@ -48,7 +48,7 @@ fn term(i: &str) -> IResult<&str, i64> {
}
},
)
.parse_next(i)
.parse_peek(i)
}

// We transform an integer string into a i64, ignoring surrounding whitespaces
Expand All @@ -65,12 +65,12 @@ fn factor(i: &str) -> IResult<&str, i64> {
)),
spaces,
)
.parse_next(i)
.parse_peek(i)
}

// We parse any expr surrounded by parens, ignoring all whitespaces around those
fn parens(i: &str) -> IResult<&str, i64> {
delimited('(', expr, ')').parse_next(i)
delimited('(', expr, ')').parse_peek(i)
}

#[test]
Expand Down
16 changes: 8 additions & 8 deletions examples/arithmetic/parser_ast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,16 +50,16 @@ pub fn expr(i: &str) -> IResult<&str, Expr> {
0..,
alt((
|i| {
let (i, add) = preceded("+", term).parse_next(i)?;
let (i, add) = preceded("+", term).parse_peek(i)?;
Ok((i, (Oper::Add, add)))
},
|i| {
let (i, sub) = preceded("-", term).parse_next(i)?;
let (i, sub) = preceded("-", term).parse_peek(i)?;
Ok((i, (Oper::Sub, sub)))
},
)),
)
.parse_next(i)?;
.parse_peek(i)?;

Ok((i, fold_exprs(initial, remainder)))
}
Expand All @@ -70,16 +70,16 @@ fn term(i: &str) -> IResult<&str, Expr> {
0..,
alt((
|i| {
let (i, mul) = preceded("*", factor).parse_next(i)?;
let (i, mul) = preceded("*", factor).parse_peek(i)?;
Ok((i, (Oper::Mul, mul)))
},
|i| {
let (i, div) = preceded("/", factor).parse_next(i)?;
let (i, div) = preceded("/", factor).parse_peek(i)?;
Ok((i, (Oper::Div, div)))
},
)),
)
.parse_next(i)?;
.parse_peek(i)?;

Ok((i, fold_exprs(initial, remainder)))
}
Expand All @@ -91,7 +91,7 @@ fn factor(i: &str) -> IResult<&str, Expr> {
.map(Expr::Value),
parens,
))
.parse_next(i)
.parse_peek(i)
}

fn parens(i: &str) -> IResult<&str, Expr> {
Expand All @@ -100,7 +100,7 @@ fn parens(i: &str) -> IResult<&str, Expr> {
delimited("(", expr.map(|e| Expr::Paren(Box::new(e))), ")"),
multispace,
)
.parse_next(i)
.parse_peek(i)
}

fn fold_exprs(initial: Expr, remainder: Vec<(Oper, Expr)>) -> Expr {
Expand Down
6 changes: 3 additions & 3 deletions examples/css/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@ impl std::str::FromStr for Color {
}

pub fn hex_color(input: &str) -> IResult<&str, Color> {
let (input, _) = "#".parse_next(input)?;
let (input, (red, green, blue)) = (hex_primary, hex_primary, hex_primary).parse_next(input)?;
let (input, _) = "#".parse_peek(input)?;
let (input, (red, green, blue)) = (hex_primary, hex_primary, hex_primary).parse_peek(input)?;

Ok((input, Color { red, green, blue }))
}

fn hex_primary(input: &str) -> IResult<&str, u8> {
take_while(2, |c: char| c.is_ascii_hexdigit())
.try_map(|input| u8::from_str_radix(input, 16))
.parse_next(input)
.parse_peek(input)
}
24 changes: 12 additions & 12 deletions examples/http/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,17 +44,17 @@ pub fn parse(data: &[u8]) -> Option<Vec<(Request<'_>, Vec<Header<'_>>)>> {

fn request(input: Stream<'_>) -> IResult<Stream<'_>, (Request<'_>, Vec<Header<'_>>)> {
let (input, req) = request_line(input)?;
let (input, h) = repeat(1.., message_header).parse_next(input)?;
let (input, h) = repeat(1.., message_header).parse_peek(input)?;
let (input, _) = line_ending(input)?;

Ok((input, (req, h)))
}

fn request_line(input: Stream<'_>) -> IResult<Stream<'_>, Request<'_>> {
let (input, method) = take_while(1.., is_token).parse_next(input)?;
let (input, _) = take_while(1.., is_space).parse_next(input)?;
let (input, uri) = take_while(1.., is_not_space).parse_next(input)?;
let (input, _) = take_while(1.., is_space).parse_next(input)?;
let (input, method) = take_while(1.., is_token).parse_peek(input)?;
let (input, _) = take_while(1.., is_space).parse_peek(input)?;
let (input, uri) = take_while(1.., is_not_space).parse_peek(input)?;
let (input, _) = take_while(1.., is_space).parse_peek(input)?;
let (input, version) = http_version(input)?;
let (input, _) = line_ending(input)?;

Expand All @@ -69,24 +69,24 @@ fn request_line(input: Stream<'_>) -> IResult<Stream<'_>, Request<'_>> {
}

fn http_version(input: Stream<'_>) -> IResult<Stream<'_>, &[u8]> {
let (input, _) = "HTTP/".parse_next(input)?;
let (input, version) = take_while(1.., is_version).parse_next(input)?;
let (input, _) = "HTTP/".parse_peek(input)?;
let (input, version) = take_while(1.., is_version).parse_peek(input)?;

Ok((input, version))
}

fn message_header_value(input: Stream<'_>) -> IResult<Stream<'_>, &[u8]> {
let (input, _) = take_while(1.., is_horizontal_space).parse_next(input)?;
let (input, data) = take_while(1.., not_line_ending).parse_next(input)?;
let (input, _) = take_while(1.., is_horizontal_space).parse_peek(input)?;
let (input, data) = take_while(1.., not_line_ending).parse_peek(input)?;
let (input, _) = line_ending(input)?;

Ok((input, data))
}

fn message_header(input: Stream<'_>) -> IResult<Stream<'_>, Header<'_>> {
let (input, name) = take_while(1.., is_token).parse_next(input)?;
let (input, _) = ':'.parse_next(input)?;
let (input, value) = repeat(1.., message_header_value).parse_next(input)?;
let (input, name) = take_while(1.., is_token).parse_peek(input)?;
let (input, _) = ':'.parse_peek(input)?;
let (input, value) = repeat(1.., message_header_value).parse_peek(input)?;

Ok((input, Header { name, value }))
}
Expand Down
24 changes: 12 additions & 12 deletions examples/http/parser_streaming.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,17 +46,17 @@ pub fn parse(data: &[u8]) -> Option<Vec<(Request<'_>, Vec<Header<'_>>)>> {

fn request(input: Stream<'_>) -> IResult<Stream<'_>, (Request<'_>, Vec<Header<'_>>)> {
let (input, req) = request_line(input)?;
let (input, h) = repeat(1.., message_header).parse_next(input)?;
let (input, h) = repeat(1.., message_header).parse_peek(input)?;
let (input, _) = line_ending(input)?;

Ok((input, (req, h)))
}

fn request_line(input: Stream<'_>) -> IResult<Stream<'_>, Request<'_>> {
let (input, method) = take_while(1.., is_token).parse_next(input)?;
let (input, _) = take_while(1.., is_space).parse_next(input)?;
let (input, uri) = take_while(1.., is_not_space).parse_next(input)?;
let (input, _) = take_while(1.., is_space).parse_next(input)?;
let (input, method) = take_while(1.., is_token).parse_peek(input)?;
let (input, _) = take_while(1.., is_space).parse_peek(input)?;
let (input, uri) = take_while(1.., is_not_space).parse_peek(input)?;
let (input, _) = take_while(1.., is_space).parse_peek(input)?;
let (input, version) = http_version(input)?;
let (input, _) = line_ending(input)?;

Expand All @@ -71,24 +71,24 @@ fn request_line(input: Stream<'_>) -> IResult<Stream<'_>, Request<'_>> {
}

fn http_version(input: Stream<'_>) -> IResult<Stream<'_>, &[u8]> {
let (input, _) = "HTTP/".parse_next(input)?;
let (input, version) = take_while(1.., is_version).parse_next(input)?;
let (input, _) = "HTTP/".parse_peek(input)?;
let (input, version) = take_while(1.., is_version).parse_peek(input)?;

Ok((input, version))
}

fn message_header_value(input: Stream<'_>) -> IResult<Stream<'_>, &[u8]> {
let (input, _) = take_while(1.., is_horizontal_space).parse_next(input)?;
let (input, data) = take_while(1.., not_line_ending).parse_next(input)?;
let (input, _) = take_while(1.., is_horizontal_space).parse_peek(input)?;
let (input, data) = take_while(1.., not_line_ending).parse_peek(input)?;
let (input, _) = line_ending(input)?;

Ok((input, data))
}

fn message_header(input: Stream<'_>) -> IResult<Stream<'_>, Header<'_>> {
let (input, name) = take_while(1.., is_token).parse_next(input)?;
let (input, _) = ':'.parse_next(input)?;
let (input, value) = repeat(1.., message_header_value).parse_next(input)?;
let (input, name) = take_while(1.., is_token).parse_peek(input)?;
let (input, _) = ':'.parse_peek(input)?;
let (input, value) = repeat(1.., message_header_value).parse_peek(input)?;

Ok((input, Header { name, value }))
}
Expand Down
2 changes: 1 addition & 1 deletion examples/ini/bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ file=payroll.dat
\0";

fn acc(i: parser::Stream<'_>) -> IResult<parser::Stream<'_>, Vec<(&str, &str)>> {
repeat(0.., parser::key_value).parse_next(i)
repeat(0.., parser::key_value).parse_peek(i)
}

let mut group = c.benchmark_group("ini keys and values");
Expand Down
12 changes: 6 additions & 6 deletions examples/ini/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,22 +21,22 @@ pub fn categories(i: Stream<'_>) -> IResult<Stream<'_>, HashMap<&str, HashMap<&s
repeat(0.., terminated(key_value, opt(multispace))),
),
)
.parse_next(i)
.parse_peek(i)
}

fn category(i: Stream<'_>) -> IResult<Stream<'_>, &str> {
delimited('[', take_while(0.., |c| c != b']'), ']')
.try_map(str::from_utf8)
.parse_next(i)
.parse_peek(i)
}

pub fn key_value(i: Stream<'_>) -> IResult<Stream<'_>, (&str, &str)> {
let (i, key) = alphanumeric.try_map(str::from_utf8).parse_next(i)?;
let (i, _) = (opt(space), '=', opt(space)).parse_next(i)?;
let (i, key) = alphanumeric.try_map(str::from_utf8).parse_peek(i)?;
let (i, _) = (opt(space), '=', opt(space)).parse_peek(i)?;
let (i, val) = take_while(0.., |c| c != b'\n' && c != b';')
.try_map(str::from_utf8)
.parse_next(i)?;
let (i, _) = opt((';', take_while(0.., |c| c != b'\n'))).parse_next(i)?;
.parse_peek(i)?;
let (i, _) = opt((';', take_while(0.., |c| c != b'\n'))).parse_peek(i)?;
Ok((i, (key, val)))
}

Expand Down
22 changes: 11 additions & 11 deletions examples/ini/parser_str.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,32 +12,32 @@ use winnow::{
pub type Stream<'i> = &'i str;

pub fn categories(input: Stream<'_>) -> IResult<Stream<'_>, HashMap<&str, HashMap<&str, &str>>> {
repeat(0.., category_and_keys).parse_next(input)
repeat(0.., category_and_keys).parse_peek(input)
}

fn category_and_keys(i: Stream<'_>) -> IResult<Stream<'_>, (&str, HashMap<&str, &str>)> {
(category, keys_and_values).parse_next(i)
(category, keys_and_values).parse_peek(i)
}

fn category(i: Stream<'_>) -> IResult<Stream<'_>, &str> {
terminated(
delimited('[', take_while(0.., |c| c != ']'), ']'),
opt(take_while(1.., [' ', '\r', '\n'])),
)
.parse_next(i)
.parse_peek(i)
}

fn keys_and_values(input: Stream<'_>) -> IResult<Stream<'_>, HashMap<&str, &str>> {
repeat(0.., key_value).parse_next(input)
repeat(0.., key_value).parse_peek(input)
}

fn key_value(i: Stream<'_>) -> IResult<Stream<'_>, (&str, &str)> {
let (i, key) = alphanumeric(i)?;
let (i, _) = (opt(space), "=", opt(space)).parse_next(i)?;
let (i, val) = take_till0(is_line_ending_or_comment).parse_next(i)?;
let (i, _) = opt(space).parse_next(i)?;
let (i, _) = opt((";", not_line_ending)).parse_next(i)?;
let (i, _) = opt(space_or_line_ending).parse_next(i)?;
let (i, _) = (opt(space), "=", opt(space)).parse_peek(i)?;
let (i, val) = take_till0(is_line_ending_or_comment).parse_peek(i)?;
let (i, _) = opt(space).parse_peek(i)?;
let (i, _) = opt((";", not_line_ending)).parse_peek(i)?;
let (i, _) = opt(space_or_line_ending).parse_peek(i)?;

Ok((i, (key, val)))
}
Expand All @@ -47,11 +47,11 @@ fn is_line_ending_or_comment(chr: char) -> bool {
}

fn not_line_ending(i: Stream<'_>) -> IResult<Stream<'_>, &str> {
take_while(0.., |c| c != '\r' && c != '\n').parse_next(i)
take_while(0.., |c| c != '\r' && c != '\n').parse_peek(i)
}

fn space_or_line_ending(i: Stream<'_>) -> IResult<Stream<'_>, &str> {
take_while(1.., [' ', '\r', '\n']).parse_next(i)
take_while(1.., [' ', '\r', '\n']).parse_peek(i)
}

#[test]
Expand Down
2 changes: 1 addition & 1 deletion examples/iterator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ fn main() {
let mut data = "abcabcabcabc";

fn parser(i: &str) -> IResult<&str, &str> {
"abc".parse_next(i)
"abc".parse_peek(i)
}

// `from_fn` (available from Rust 1.34) can create an iterator
Expand Down
Loading

0 comments on commit c4d3cee

Please sign in to comment.