Skip to content

Commit

Permalink
syntax and automata: bump LookSet representation from u16 to u32
Browse files Browse the repository at this point in the history
This is in preparation for adding 8 new word boundary look-around
assertions: \b{start}, \b{end}, \b{start-half} and \b{end-half}, along
with Unicode and ASCII-only variants of each.

Ref #469
  • Loading branch information
BurntSushi committed Oct 8, 2023
1 parent c8fe16b commit 9d5390f
Show file tree
Hide file tree
Showing 6 changed files with 55 additions and 50 deletions.
8 changes: 4 additions & 4 deletions regex-automata/src/dfa/dense.rs
Original file line number Diff line number Diff line change
Expand Up @@ -882,20 +882,20 @@ impl Config {
/// # if !cfg!(target_pointer_width = "64") { return Ok(()); } // see #1039
/// use regex_automata::{dfa::{dense, Automaton}, Input};
///
/// // 600KB isn't enough!
/// // 700KB isn't enough!
/// dense::Builder::new()
/// .configure(dense::Config::new()
/// .determinize_size_limit(Some(600_000))
/// .determinize_size_limit(Some(700_000))
/// )
/// .build(r"\w{20}")
/// .unwrap_err();
///
/// // ... but 700KB probably is!
/// // ... but 800KB probably is!
/// // (Note that auxiliary storage sizes aren't necessarily stable between
/// // releases.)
/// let dfa = dense::Builder::new()
/// .configure(dense::Config::new()
/// .determinize_size_limit(Some(700_000))
/// .determinize_size_limit(Some(800_000))
/// )
/// .build(r"\w{20}")?;
/// let haystack = "A".repeat(20).into_bytes();
Expand Down
2 changes: 1 addition & 1 deletion regex-automata/src/dfa/onepass.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2815,7 +2815,7 @@ impl Epsilons {

/// Return the set of look-around assertions in these epsilon transitions.
fn looks(self) -> LookSet {
LookSet { bits: (self.0 & Epsilons::LOOK_MASK).low_u16() }
LookSet { bits: (self.0 & Epsilons::LOOK_MASK).low_u32() }
}

/// Set the look-around assertions on these epsilon transitions.
Expand Down
39 changes: 20 additions & 19 deletions regex-automata/src/util/determinize/state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ impl StateBuilderEmpty {
}

pub(crate) fn into_matches(mut self) -> StateBuilderMatches {
self.0.extend_from_slice(&[0, 0, 0, 0, 0]);
self.0.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0]);
StateBuilderMatches(self.0)
}

Expand Down Expand Up @@ -348,16 +348,17 @@ impl StateBuilderNFA {
/// generated by a transition over a "word" byte. (Callers may not always set
/// this. For example, if the NFA has no word boundary assertion, then needing
/// to track whether a state came from a word byte or not is superfluous and
/// wasteful.)
/// wasteful.) Bit 3 is set to 1 if the state was generated by a transition
/// from a `\r` (forward search) or a `\n` (reverse search) when CRLF mode is
/// enabled.
///
/// Byte 1 corresponds to the look-behind assertions that were satisfied by
/// the transition that created this state. This generally only includes the
/// StartLF and Start assertions. (Look-ahead assertions are not tracked as
/// part of states. Instead, these are applied by re-computing the epsilon
/// closure of a state when computing the transition function. See `next` in
/// the parent module.)
/// Bytes 1..5 correspond to the look-behind assertions that were satisfied
/// by the transition that created this state. (Look-ahead assertions are not
/// tracked as part of states. Instead, these are applied by re-computing the
/// epsilon closure of a state when computing the transition function. See
/// `next` in the parent module.)
///
/// Byte 2 corresponds to the set of look-around assertions (including both
/// Bytes 5..9 correspond to the set of look-around assertions (including both
/// look-behind and look-ahead) that appear somewhere in this state's set of
/// NFA state IDs. This is used to determine whether this state's epsilon
/// closure should be re-computed when computing the transition function.
Expand All @@ -366,7 +367,7 @@ impl StateBuilderNFA {
/// function, we should only re-compute the epsilon closure if those new
/// assertions are relevant to this particular state.
///
/// Bytes 3..7 correspond to a 32-bit native-endian encoded integer
/// Bytes 9..13 correspond to a 32-bit native-endian encoded integer
/// corresponding to the number of patterns encoded in this state. If the state
/// is not a match state (byte 0 bit 0 is 0) or if it's only pattern ID is
/// PatternID::ZERO, then no integer is encoded at this position. Instead, byte
Expand Down Expand Up @@ -452,7 +453,7 @@ impl<'a> Repr<'a> {
/// state has no conditional epsilon transitions, then there is no need
/// to re-compute the epsilon closure.
fn look_need(&self) -> LookSet {
LookSet::read_repr(&self.0[3..])
LookSet::read_repr(&self.0[5..])
}

/// Returns the total number of match pattern IDs in this state.
Expand All @@ -476,7 +477,7 @@ impl<'a> Repr<'a> {
if !self.has_pattern_ids() {
PatternID::ZERO
} else {
let offset = 9 + index * PatternID::SIZE;
let offset = 13 + index * PatternID::SIZE;
// This is OK since we only ever serialize valid PatternIDs to
// states.
wire::read_pattern_id_unchecked(&self.0[offset..]).0
Expand Down Expand Up @@ -507,7 +508,7 @@ impl<'a> Repr<'a> {
f(PatternID::ZERO);
return;
}
let mut pids = &self.0[9..self.pattern_offset_end()];
let mut pids = &self.0[13..self.pattern_offset_end()];
while !pids.is_empty() {
let pid = wire::read_u32(pids);
pids = &pids[PatternID::SIZE..];
Expand Down Expand Up @@ -539,11 +540,11 @@ impl<'a> Repr<'a> {
fn pattern_offset_end(&self) -> usize {
let encoded = self.encoded_pattern_len();
if encoded == 0 {
return 5;
return 9;
}
// This arithmetic is OK since we were able to address this many bytes
// when writing to the state, thus, it must fit into a usize.
encoded.checked_mul(4).unwrap().checked_add(9).unwrap()
encoded.checked_mul(4).unwrap().checked_add(13).unwrap()
}

/// Returns the total number of *encoded* pattern IDs in this state.
Expand All @@ -557,7 +558,7 @@ impl<'a> Repr<'a> {
}
// This unwrap is OK since the total number of patterns is always
// guaranteed to fit into a usize.
usize::try_from(wire::read_u32(&self.0[5..9])).unwrap()
usize::try_from(wire::read_u32(&self.0[9..13])).unwrap()
}
}

Expand Down Expand Up @@ -643,7 +644,7 @@ impl<'a> ReprVec<'a> {
/// Mutate the set of look-around (both behind and ahead) assertions that
/// appear at least once in this state's set of NFA states.
fn set_look_need(&mut self, mut set: impl FnMut(LookSet) -> LookSet) {
set(self.look_need()).write_repr(&mut self.0[3..]);
set(self.look_need()).write_repr(&mut self.0[5..]);
}

/// Add a pattern ID to this state. All match states must have at least
Expand Down Expand Up @@ -703,14 +704,14 @@ impl<'a> ReprVec<'a> {
return;
}
let patsize = PatternID::SIZE;
let pattern_bytes = self.0.len() - 9;
let pattern_bytes = self.0.len() - 13;
// Every pattern ID uses 4 bytes, so number of bytes should be
// divisible by 4.
assert_eq!(pattern_bytes % patsize, 0);
// This unwrap is OK since we are guaranteed that the maximum number
// of possible patterns fits into a u32.
let count32 = u32::try_from(pattern_bytes / patsize).unwrap();
wire::NE::write_u32(count32, &mut self.0[5..9]);
wire::NE::write_u32(count32, &mut self.0[9..13]);
}

/// Add an NFA state ID to this state. The order in which NFA states are
Expand Down
26 changes: 14 additions & 12 deletions regex-automata/src/util/look.rs
Original file line number Diff line number Diff line change
Expand Up @@ -125,17 +125,17 @@ impl Look {
/// constructor is guaranteed to return the same look-around variant that
/// one started with within a semver compatible release of this crate.
#[inline]
pub const fn as_repr(self) -> u16 {
pub const fn as_repr(self) -> u32 {
// AFAIK, 'as' is the only way to zero-cost convert an int enum to an
// actual int.
self as u16
self as u32
}

/// Given the underlying representation of a `Look` value, return the
/// corresponding `Look` value if the representation is valid. Otherwise
/// `None` is returned.
#[inline]
pub const fn from_repr(repr: u16) -> Option<Look> {
pub const fn from_repr(repr: u32) -> Option<Look> {
match repr {
0b00_0000_0001 => Some(Look::Start),
0b00_0000_0010 => Some(Look::End),
Expand Down Expand Up @@ -191,7 +191,7 @@ pub struct LookSet {
/// range of `u16` values to be represented. For example, even if the
/// current implementation only makes use of the 10 least significant bits,
/// it may use more bits in a future semver compatible release.
pub bits: u16,
pub bits: u32,
}

impl LookSet {
Expand Down Expand Up @@ -379,29 +379,31 @@ impl LookSet {
*self = self.intersect(other);
}

/// Return a `LookSet` from the slice given as a native endian 16-bit
/// Return a `LookSet` from the slice given as a native endian 32-bit
/// integer.
///
/// # Panics
///
/// This panics if `slice.len() < 2`.
/// This panics if `slice.len() < 4`.
#[inline]
pub fn read_repr(slice: &[u8]) -> LookSet {
let bits = u16::from_ne_bytes(slice[..2].try_into().unwrap());
let bits = u32::from_ne_bytes(slice[..4].try_into().unwrap());
LookSet { bits }
}

/// Write a `LookSet` as a native endian 16-bit integer to the beginning
/// Write a `LookSet` as a native endian 32-bit integer to the beginning
/// of the slice given.
///
/// # Panics
///
/// This panics if `slice.len() < 2`.
/// This panics if `slice.len() < 4`.
#[inline]
pub fn write_repr(self, slice: &mut [u8]) {
let raw = self.bits.to_ne_bytes();
slice[0] = raw[0];
slice[1] = raw[1];
slice[2] = raw[2];
slice[3] = raw[3];
}

/// Checks that all assertions in this set can be matched.
Expand Down Expand Up @@ -456,9 +458,9 @@ impl Iterator for LookSetIter {
return None;
}
// We'll never have more than u8::MAX distinct look-around assertions,
// so 'repr' will always fit into a u16.
let repr = u16::try_from(self.set.bits.trailing_zeros()).unwrap();
let look = Look::from_repr(1 << repr)?;
// so 'bit' will always fit into a u16.
let bit = u16::try_from(self.set.bits.trailing_zeros()).unwrap();
let look = Look::from_repr(1 << bit)?;
self.set = self.set.remove(look);
Some(look)
}
Expand Down
4 changes: 2 additions & 2 deletions regex-automata/tests/hybrid/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ fn too_many_cache_resets_cause_quit() -> Result<(), Box<dyn Error>> {
let mut cache = dfa.create_cache();

let haystack = "a".repeat(101).into_bytes();
let err = MatchError::gave_up(25);
let err = MatchError::gave_up(24);
// Notice that we make the same amount of progress in each search! That's
// because the cache is reused and already has states to handle the first
// N bytes.
Expand Down Expand Up @@ -83,7 +83,7 @@ fn too_many_cache_resets_cause_quit() -> Result<(), Box<dyn Error>> {
// OK, if we reset the cache, then we should be able to create more states
// and make more progress with searching for betas.
cache.reset(&dfa);
let err = MatchError::gave_up(27);
let err = MatchError::gave_up(26);
assert_eq!(
Err(err),
dfa.try_search_fwd(&mut cache, &Input::new(&haystack))
Expand Down
26 changes: 14 additions & 12 deletions regex-syntax/src/hir/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1664,17 +1664,17 @@ impl Look {
/// constructor is guaranteed to return the same look-around variant that
/// one started with within a semver compatible release of this crate.
#[inline]
pub const fn as_repr(self) -> u16 {
pub const fn as_repr(self) -> u32 {
// AFAIK, 'as' is the only way to zero-cost convert an int enum to an
// actual int.
self as u16
self as u32
}

/// Given the underlying representation of a `Look` value, return the
/// corresponding `Look` value if the representation is valid. Otherwise
/// `None` is returned.
#[inline]
pub const fn from_repr(repr: u16) -> Option<Look> {
pub const fn from_repr(repr: u32) -> Option<Look> {
match repr {
0b00_0000_0001 => Some(Look::Start),
0b00_0000_0010 => Some(Look::End),
Expand Down Expand Up @@ -2600,7 +2600,7 @@ pub struct LookSet {
/// range of `u16` values to be represented. For example, even if the
/// current implementation only makes use of the 10 least significant bits,
/// it may use more bits in a future semver compatible release.
pub bits: u16,
pub bits: u32,
}

impl LookSet {
Expand Down Expand Up @@ -2788,29 +2788,31 @@ impl LookSet {
*self = self.intersect(other);
}

/// Return a `LookSet` from the slice given as a native endian 16-bit
/// Return a `LookSet` from the slice given as a native endian 32-bit
/// integer.
///
/// # Panics
///
/// This panics if `slice.len() < 2`.
/// This panics if `slice.len() < 4`.
#[inline]
pub fn read_repr(slice: &[u8]) -> LookSet {
let bits = u16::from_ne_bytes(slice[..2].try_into().unwrap());
let bits = u32::from_ne_bytes(slice[..4].try_into().unwrap());
LookSet { bits }
}

/// Write a `LookSet` as a native endian 16-bit integer to the beginning
/// Write a `LookSet` as a native endian 32-bit integer to the beginning
/// of the slice given.
///
/// # Panics
///
/// This panics if `slice.len() < 2`.
/// This panics if `slice.len() < 4`.
#[inline]
pub fn write_repr(self, slice: &mut [u8]) {
let raw = self.bits.to_ne_bytes();
slice[0] = raw[0];
slice[1] = raw[1];
slice[2] = raw[2];
slice[3] = raw[3];
}
}

Expand Down Expand Up @@ -2843,9 +2845,9 @@ impl Iterator for LookSetIter {
return None;
}
// We'll never have more than u8::MAX distinct look-around assertions,
// so 'repr' will always fit into a u16.
let repr = u16::try_from(self.set.bits.trailing_zeros()).unwrap();
let look = Look::from_repr(1 << repr)?;
// so 'bit' will always fit into a u16.
let bit = u16::try_from(self.set.bits.trailing_zeros()).unwrap();
let look = Look::from_repr(1 << bit)?;
self.set = self.set.remove(look);
Some(look)
}
Expand Down

0 comments on commit 9d5390f

Please sign in to comment.