From ba3b9eaeb910fea237026db6708828c904e05fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jergu=C5=A1=20Lejko?= Date: Tue, 4 Aug 2026 13:47:22 +0200 Subject: [PATCH] automata: make the PikeVM optional in the meta engine --- regex-automata/src/meta/error.rs | 8 +- regex-automata/src/meta/regex.rs | 227 +++++++++++++- regex-automata/src/meta/strategy.rs | 467 ++++++++++++++++++---------- regex-automata/src/meta/wrappers.rs | 38 ++- regex-automata/src/util/search.rs | 24 ++ 5 files changed, 584 insertions(+), 180 deletions(-) diff --git a/regex-automata/src/meta/error.rs b/regex-automata/src/meta/error.rs index 9ead729bbd..966e64b248 100644 --- a/regex-automata/src/meta/error.rs +++ b/regex-automata/src/meta/error.rs @@ -205,6 +205,10 @@ impl RetryFailError { pub(crate) fn from_offset(offset: usize) -> RetryFailError { RetryFailError { offset } } + + pub(crate) fn offset(&self) -> usize { + self.offset + } } #[cfg(feature = "std")] @@ -233,7 +237,9 @@ impl From for RetryFailError { // or with higher level control flow logic. For example, the // backtracker's wrapper will never hand out a backtracker engine // when the haystack would be too long. - HaystackTooLong { .. } | UnsupportedAnchored { .. } => { + HaystackTooLong { .. } + | UnsupportedAnchored { .. } + | NoEngine { .. } => { unreachable!("found impossible error in meta engine: {merr}") } } diff --git a/regex-automata/src/meta/regex.rs b/regex-automata/src/meta/regex.rs index 7e25763ad1..4c6470cb53 100644 --- a/regex-automata/src/meta/regex.rs +++ b/regex-automata/src/meta/regex.rs @@ -24,7 +24,9 @@ use crate::{ pool::{Pool, PoolGuard}, prefilter::Prefilter, primitives::{NonMaxUsize, PatternID}, - search::{HalfMatch, Input, Match, MatchKind, PatternSet, Span}, + search::{ + HalfMatch, Input, Match, MatchError, MatchKind, PatternSet, Span, + }, }, }; @@ -540,6 +542,61 @@ impl Regex { result } + /// Executes a leftmost is-match search, returning an error instead of + /// falling back to the PikeVM. + /// + /// This is like [`Regex::is_match`], but for a `Regex` built with + /// [`Config::pikevm`] disabled. The meta regex engine's fast engines (the + /// DFAs) can give up on a search, either because a lazy DFA's cache is + /// being used ineffectively or because a Unicode word boundary was seen + /// alongside a non-ASCII byte. The engines with bounded runtime (the + /// one-pass DFA and the bounded backtracker) still pick those up, and with + /// the PikeVM disabled the backtracker is given every search it can serve + /// rather than only the short haystacks it is normally reserved for. Only + /// once nothing is left to run does this routine report a [`MatchError`], + /// which is to say only when the haystack outgrows the backtracker's + /// visited bitset (see [`Config::backtrack_visited_capacity`]). + /// + /// A `Regex` built with the PikeVM enabled (the default) never returns an + /// error here. + /// + /// # Example + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::builder() + /// .configure(Regex::config().pikevm(false)) + /// .build(r"\b\w+\b")?; + /// let mut cache = re.create_cache(); + /// + /// assert_eq!(Ok(true), re.try_is_match_with(&mut cache, "quux")); + /// // A Unicode word boundary against a non-ASCII haystack makes the lazy + /// // DFA quit, but the backtracker still answers it. + /// let haystack = "☃".repeat(100); + /// assert_eq!(Ok(false), re.try_is_match_with(&mut cache, &haystack)); + /// // Past the backtracker's bound there is no engine left to run. + /// let haystack = "☃".repeat(10_000); + /// assert!(re.try_is_match_with(&mut cache, &haystack).is_err()); + /// + /// # Ok::<(), Box>(()) + /// ``` + #[inline] + pub fn try_is_match_with<'h, I: Into>>( + &self, + cache: &mut Cache, + input: I, + ) -> Result { + let input = input.into().earliest(true); + if self.imp.info.is_impossible(&input) { + return Ok(false); + } + self.imp + .strat + .try_is_match(cache, &input) + .map_err(|err| MatchError::no_engine(err.offset())) + } + /// Executes a leftmost search and returns the first match that is found, /// if one exists. /// @@ -1257,6 +1314,30 @@ impl Regex { self.imp.strat.search(cache, input) } + /// This is like [`Regex::search_with`], but returns an error instead of + /// falling back to the PikeVM. + /// + /// A `Regex` built with [`Config::pikevm`] disabled has no engine left + /// once the DFAs and the bounded engines have all given up on a search. + /// This routine reports that as a [`MatchError`], where the infallible + /// variants panic. A `Regex` built with the PikeVM enabled (the default) + /// never returns an error here. + pub fn try_search_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result, MatchError> { + if self.imp.info.captures_disabled() + || self.imp.info.is_impossible(input) + { + return Ok(None); + } + self.imp + .strat + .try_search(cache, input) + .map_err(|err| MatchError::no_engine(err.offset())) + } + /// This is like [`Regex::search_half`], but requires the caller to /// explicitly pass a [`Cache`]. /// @@ -1301,6 +1382,30 @@ impl Regex { self.imp.strat.search_half(cache, input) } + /// This is like [`Regex::search_half_with`], but returns an error instead of + /// falling back to the PikeVM. + /// + /// A `Regex` built with [`Config::pikevm`] disabled has no engine left + /// once the DFAs and the bounded engines have all given up on a search. + /// This routine reports that as a [`MatchError`], where the infallible + /// variants panic. A `Regex` built with the PikeVM enabled (the default) + /// never returns an error here. + pub fn try_search_half_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result, MatchError> { + if self.imp.info.captures_disabled() + || self.imp.info.is_impossible(input) + { + return Ok(None); + } + self.imp + .strat + .try_search_half(cache, input) + .map_err(|err| MatchError::no_engine(err.offset())) + } + /// This is like [`Regex::search_captures`], but requires the caller to /// explicitly pass a [`Cache`]. /// @@ -1394,6 +1499,27 @@ impl Regex { caps.set_pattern(pid); } + /// This is like [`Regex::search_captures_with`], but returns an error instead of + /// falling back to the PikeVM. + /// + /// A `Regex` built with [`Config::pikevm`] disabled has no engine left + /// once the DFAs and the bounded engines have all given up on a search. + /// This routine reports that as a [`MatchError`], where the infallible + /// variants panic. A `Regex` built with the PikeVM enabled (the default) + /// never returns an error here. + pub fn try_search_captures_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + caps: &mut Captures, + ) -> Result<(), MatchError> { + caps.set_pattern(None); + let pid = + self.try_search_slots_with(cache, input, caps.slots_mut())?; + caps.set_pattern(pid); + Ok(()) + } + /// This is like [`Regex::search_slots`], but requires the caller to /// explicitly pass a [`Cache`]. /// @@ -1456,6 +1582,31 @@ impl Regex { self.imp.strat.search_slots(cache, input, slots) } + /// This is like [`Regex::search_slots_with`], but returns an error instead of + /// falling back to the PikeVM. + /// + /// A `Regex` built with [`Config::pikevm`] disabled has no engine left + /// once the DFAs and the bounded engines have all given up on a search. + /// This routine reports that as a [`MatchError`], where the infallible + /// variants panic. A `Regex` built with the PikeVM enabled (the default) + /// never returns an error here. + pub fn try_search_slots_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option], + ) -> Result, MatchError> { + if self.imp.info.captures_disabled() + || self.imp.info.is_impossible(input) + { + return Ok(None); + } + self.imp + .strat + .try_search_slots(cache, input, slots) + .map_err(|err| MatchError::no_engine(err.offset())) + } + /// This is like [`Regex::which_overlapping_matches`], but requires the /// caller to explicitly pass a [`Cache`]. /// @@ -1504,6 +1655,29 @@ impl Regex { } self.imp.strat.which_overlapping_matches(cache, input, patset) } + + /// This is like [`Regex::which_overlapping_matches_with`], but returns an error instead of + /// falling back to the PikeVM. + /// + /// A `Regex` built with [`Config::pikevm`] disabled has no engine left + /// once the DFAs and the bounded engines have all given up on a search. + /// This routine reports that as a [`MatchError`], where the infallible + /// variants panic. A `Regex` built with the PikeVM enabled (the default) + /// never returns an error here. + pub fn try_which_overlapping_matches_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) -> Result<(), MatchError> { + if self.imp.info.is_impossible(input) { + return Ok(()); + } + self.imp + .strat + .try_which_overlapping_matches(cache, input, patset) + .map_err(|err| MatchError::no_engine(err.offset())) + } } /// Various non-search routines for querying properties of a `Regex` and @@ -2481,6 +2655,8 @@ pub struct Config { dfa_state_limit: Option>, onepass: Option, backtrack: Option, + backtrack_visited_capacity: Option, + pikevm: Option, byte_classes: Option, line_terminator: Option, } @@ -3097,6 +3273,39 @@ impl Config { Config { backtrack: Some(yes), ..self } } + /// Sets the visited capacity, in bytes, used by the bounded backtracker. + /// + /// The backtracker refuses to run when the haystack is longer than its + /// visited bitset can track, which is roughly this capacity divided by the + /// number of NFA states. Raising it lets the backtracker serve longer + /// haystacks, which matters when [`Config::pikevm`] is disabled and the + /// backtracker is the last engine standing. + /// + /// When unset, the backtracker's own default is used. + pub fn backtrack_visited_capacity(self, capacity: usize) -> Config { + Config { backtrack_visited_capacity: Some(capacity), ..self } + } + + /// Whether to permit the use of the PikeVM. + /// + /// The PikeVM is the meta regex engine's engine of last resort: it can + /// handle any regex against any haystack, but its NFA simulation can be + /// orders of magnitude slower than the DFAs. It is what makes the + /// infallible search routines infallible. + /// + /// Disabling it turns this into a regex engine that can fail: a search + /// that the faster engines give up on has nothing left to run it. The + /// `try_` search routines report that condition as a [`MatchError`] with + /// a [`MatchErrorKind::NoEngine`](crate::MatchErrorKind::NoEngine) kind, + /// while the infallible routines panic. This is useful for rejecting a + /// pathological regex outright rather than paying for it. + /// + /// This is enabled by default, and when it is, no search routine can + /// fail or panic. + pub fn pikevm(self, yes: bool) -> Config { + Config { pikevm: Some(yes), ..self } + } + /// Returns the match kind on this configuration, as set by /// [`Config::match_kind`]. /// @@ -3270,6 +3479,18 @@ impl Config { } } + /// Returns the backtracker's visited capacity, as set by + /// [`Config::backtrack_visited_capacity`]. `None` means the backtracker's + /// own default is used. + pub fn get_backtrack_visited_capacity(&self) -> Option { + self.backtrack_visited_capacity + } + + /// Returns whether the PikeVM may be used, as set by [`Config::pikevm`]. + pub fn get_pikevm(&self) -> bool { + self.pikevm.unwrap_or(true) + } + /// Returns a "baseline" Thompson configuration for constructing NFAs based /// on this configuration. /// @@ -3315,6 +3536,10 @@ impl Config { dfa_state_limit: o.dfa_state_limit.or(self.dfa_state_limit), onepass: o.onepass.or(self.onepass), backtrack: o.backtrack.or(self.backtrack), + backtrack_visited_capacity: o + .backtrack_visited_capacity + .or(self.backtrack_visited_capacity), + pikevm: o.pikevm.or(self.pikevm), byte_classes: o.byte_classes.or(self.byte_classes), line_terminator: o.line_terminator.or(self.line_terminator), } diff --git a/regex-automata/src/meta/strategy.rs b/regex-automata/src/meta/strategy.rs index c121443039..36e2c8548c 100644 --- a/regex-automata/src/meta/strategy.rs +++ b/regex-automata/src/meta/strategy.rs @@ -53,31 +53,82 @@ pub(super) trait Strategy: fn memory_usage(&self) -> usize; - fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option; + // The infallible half of each search routine. Only a regex built with + // 'Config::pikevm' disabled can reach the panic, and then only for a + // search the remaining engines all gave up on. + fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option { + self.try_search(cache, input).expect(PIKEVM_DISABLED) + } fn search_half( &self, cache: &mut Cache, input: &Input<'_>, - ) -> Option; + ) -> Option { + self.try_search_half(cache, input).expect(PIKEVM_DISABLED) + } - fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool; + fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + self.try_is_match(cache, input).expect(PIKEVM_DISABLED) + } fn search_slots( &self, cache: &mut Cache, input: &Input<'_>, slots: &mut [Option], - ) -> Option; + ) -> Option { + self.try_search_slots(cache, input, slots).expect(PIKEVM_DISABLED) + } fn which_overlapping_matches( &self, cache: &mut Cache, input: &Input<'_>, patset: &mut PatternSet, - ); + ) { + self.try_which_overlapping_matches(cache, input, patset) + .expect(PIKEVM_DISABLED) + } + + // Each of these returns an error only when the search needed the PikeVM + // but it was disabled via 'Config::pikevm'. + fn try_search( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result, RetryFailError>; + + fn try_search_half( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result, RetryFailError>; + + fn try_is_match( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result; + + fn try_search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option], + ) -> Result, RetryFailError>; + + fn try_which_overlapping_matches( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) -> Result<(), RetryFailError>; } +const PIKEVM_DISABLED: &str = + "PikeVM is disabled, use the try_ variant of this search"; + pub(super) fn new( info: &RegexInfo, hirs: &[&Hir], @@ -389,62 +440,77 @@ impl Strategy for Pre

{ } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search(&self, _cache: &mut Cache, input: &Input<'_>) -> Option { + fn try_search( + &self, + _cache: &mut Cache, + input: &Input<'_>, + ) -> Result, RetryFailError> { if input.is_done() { - return None; + return Ok(None); } if input.get_anchored().is_anchored() { - return self + return Ok(self .pre .prefix(input.haystack(), input.get_span()) - .map(|sp| Match::new(PatternID::ZERO, sp)); + .map(|sp| Match::new(PatternID::ZERO, sp))); } - self.pre + Ok(self + .pre .find(input.haystack(), input.get_span()) - .map(|sp| Match::new(PatternID::ZERO, sp)) + .map(|sp| Match::new(PatternID::ZERO, sp))) } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_half( + fn try_search_half( &self, cache: &mut Cache, input: &Input<'_>, - ) -> Option { - self.search(cache, input).map(|m| HalfMatch::new(m.pattern(), m.end())) + ) -> Result, RetryFailError> { + Ok(self + .try_search(cache, input)? + .map(|m| HalfMatch::new(m.pattern(), m.end()))) } #[cfg_attr(feature = "perf-inline", inline(always))] - fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { - self.search(cache, input).is_some() + fn try_is_match( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result { + Ok(self.try_search(cache, input)?.is_some()) } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_slots( + fn try_search_slots( &self, cache: &mut Cache, input: &Input<'_>, slots: &mut [Option], - ) -> Option { - let m = self.search(cache, input)?; + ) -> Result, RetryFailError> { + let m = match self.try_search(cache, input)? { + None => return Ok(None), + Some(m) => m, + }; if let Some(slot) = slots.get_mut(0) { *slot = NonMaxUsize::new(m.start()); } if let Some(slot) = slots.get_mut(1) { *slot = NonMaxUsize::new(m.end()); } - Some(m.pattern()) + Ok(Some(m.pattern())) } #[cfg_attr(feature = "perf-inline", inline(always))] - fn which_overlapping_matches( + fn try_which_overlapping_matches( &self, cache: &mut Cache, input: &Input<'_>, patset: &mut PatternSet, - ) { - if self.search(cache, input).is_some() { + ) -> Result<(), RetryFailError> { + if self.try_search(cache, input)?.is_some() { patset.insert(PatternID::ZERO); } + Ok(()) } } @@ -566,14 +632,14 @@ impl Core { } } - fn search_nofail( + fn try_search_fallback( &self, cache: &mut Cache, input: &Input<'_>, - ) -> Option { + ) -> Result, RetryFailError> { let caps = &mut cache.capmatches; caps.set_pattern(None); - // We manually inline 'try_search_slots_nofail' here because we need to + // We manually inline 'try_search_slots_fallback' here because we need to // borrow from 'cache.capmatches' in this method, but if we do, then // we can't pass 'cache' wholesale to to 'try_slots_no_hybrid'. It's a // classic example of how the borrow checker inhibits decomposition. @@ -590,74 +656,88 @@ impl Core { e.search_slots(&mut cache.backtrack, input, caps.slots_mut()) } else { trace!("using PikeVM for search at {:?}", input.get_span()); - let e = self.pikevm.get(); + let e = self + .pikevm + .get() + .ok_or_else(|| RetryFailError::from_offset(input.start()))?; e.search_slots(&mut cache.pikevm, input, caps.slots_mut()) }; caps.set_pattern(pid); - caps.get_match() + Ok(caps.get_match()) } - fn search_half_nofail( + fn try_search_half_fallback( &self, cache: &mut Cache, input: &Input<'_>, - ) -> Option { + ) -> Result, RetryFailError> { // Only the lazy/full DFA returns half-matches, since the DFA requires // a reverse scan to find the start position. These fallback regex // engines can find the start and end in a single pass, so we just do // that and throw away the start offset to conform to the API. - let m = self.search_nofail(cache, input)?; - Some(HalfMatch::new(m.pattern(), m.end())) + Ok(self + .try_search_fallback(cache, input)? + .map(|m| HalfMatch::new(m.pattern(), m.end()))) } - fn search_slots_nofail( + fn try_search_slots_fallback( &self, cache: &mut Cache, input: &Input<'_>, slots: &mut [Option], - ) -> Option { + ) -> Result, RetryFailError> { if let Some(ref e) = self.onepass.get(input) { trace!( "using OnePass for capture search at {:?}", input.get_span() ); - e.search_slots(&mut cache.onepass, input, slots) + Ok(e.search_slots(&mut cache.onepass, input, slots)) } else if let Some(ref e) = self.backtrack.get(input) { trace!( "using BoundedBacktracker for capture search at {:?}", input.get_span() ); - e.search_slots(&mut cache.backtrack, input, slots) + Ok(e.search_slots(&mut cache.backtrack, input, slots)) } else { trace!( "using PikeVM for capture search at {:?}", input.get_span() ); - let e = self.pikevm.get(); - e.search_slots(&mut cache.pikevm, input, slots) + let e = self + .pikevm + .get() + .ok_or_else(|| RetryFailError::from_offset(input.start()))?; + Ok(e.search_slots(&mut cache.pikevm, input, slots)) } } - fn is_match_nofail(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + fn try_is_match_fallback( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result { if let Some(ref e) = self.onepass.get(input) { trace!( "using OnePass for is-match search at {:?}", input.get_span() ); - e.search_slots(&mut cache.onepass, input, &mut []).is_some() + Ok(e.search_slots(&mut cache.onepass, input, &mut []).is_some()) } else if let Some(ref e) = self.backtrack.get(input) { trace!( "using BoundedBacktracker for is-match search at {:?}", input.get_span() ); - e.is_match(&mut cache.backtrack, input) + Ok(e.is_match(&mut cache.backtrack, input)) } else { trace!( "using PikeVM for is-match search at {:?}", input.get_span() ); - let e = self.pikevm.get(); - e.is_match(&mut cache.pikevm, input) + let e = self + .pikevm + .get() + .ok_or_else(|| RetryFailError::from_offset(input.start()))?; + Ok(e.is_match(&mut cache.pikevm, input)) } } @@ -710,76 +790,84 @@ impl Strategy for Core { } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option { + fn try_search( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result, RetryFailError> { // We manually inline try_search_mayfail here because letting the // compiler do it seems to produce pretty crappy codegen. return if let Some(e) = self.dfa.get(input) { trace!("using full DFA for full search at {:?}", input.get_span()); match e.try_search(input) { - Ok(x) => x, + Ok(x) => Ok(x), Err(_err) => { trace!("full DFA search failed: {_err}"); - self.search_nofail(cache, input) + self.try_search_fallback(cache, input) } } } else if let Some(e) = self.hybrid.get(input) { trace!("using lazy DFA for full search at {:?}", input.get_span()); match e.try_search(&mut cache.hybrid, input) { - Ok(x) => x, + Ok(x) => Ok(x), Err(_err) => { trace!("lazy DFA search failed: {_err}"); - self.search_nofail(cache, input) + self.try_search_fallback(cache, input) } } } else { - self.search_nofail(cache, input) + self.try_search_fallback(cache, input) }; } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_half( + fn try_search_half( &self, cache: &mut Cache, input: &Input<'_>, - ) -> Option { + ) -> Result, RetryFailError> { // The main difference with 'search' is that if we're using a DFA, we // can use a single forward scan without needing to run the reverse // DFA. if let Some(e) = self.dfa.get(input) { trace!("using full DFA for half search at {:?}", input.get_span()); match e.try_search_half_fwd(input) { - Ok(x) => x, + Ok(x) => Ok(x), Err(_err) => { trace!("full DFA half search failed: {_err}"); - self.search_half_nofail(cache, input) + self.try_search_half_fallback(cache, input) } } } else if let Some(e) = self.hybrid.get(input) { trace!("using lazy DFA for half search at {:?}", input.get_span()); match e.try_search_half_fwd(&mut cache.hybrid, input) { - Ok(x) => x, + Ok(x) => Ok(x), Err(_err) => { trace!("lazy DFA half search failed: {_err}"); - self.search_half_nofail(cache, input) + self.try_search_half_fallback(cache, input) } } } else { - self.search_half_nofail(cache, input) + self.try_search_half_fallback(cache, input) } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + fn try_is_match( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result { if let Some(e) = self.dfa.get(input) { trace!( "using full DFA for is-match search at {:?}", input.get_span() ); match e.try_search_half_fwd(input) { - Ok(x) => x.is_some(), + Ok(x) => Ok(x.is_some()), Err(_err) => { trace!("full DFA half search failed: {_err}"); - self.is_match_nofail(cache, input) + self.try_is_match_fallback(cache, input) } } } else if let Some(e) = self.hybrid.get(input) { @@ -788,24 +876,24 @@ impl Strategy for Core { input.get_span() ); match e.try_search_half_fwd(&mut cache.hybrid, input) { - Ok(x) => x.is_some(), + Ok(x) => Ok(x.is_some()), Err(_err) => { trace!("lazy DFA half search failed: {_err}"); - self.is_match_nofail(cache, input) + self.try_is_match_fallback(cache, input) } } } else { - self.is_match_nofail(cache, input) + self.try_is_match_fallback(cache, input) } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_slots( + fn try_search_slots( &self, cache: &mut Cache, input: &Input<'_>, slots: &mut [Option], - ) -> Option { + ) -> Result, RetryFailError> { // Even if the regex has explicit capture groups, if the caller didn't // provide any explicit slots, then it doesn't make sense to try and do // extra work to get offsets for those slots. Ideally the caller should @@ -813,9 +901,12 @@ impl Strategy for Core { // we try to save the caller from themselves if they do. if !self.is_capture_search_needed(slots.len()) { trace!("asked for slots unnecessarily, trying fast path"); - let m = self.search(cache, input)?; + let m = match self.try_search(cache, input)? { + None => return Ok(None), + Some(m) => m, + }; copy_match_to_slots(m, slots); - return Some(m.pattern()); + return Ok(Some(m.pattern())); } // If the onepass DFA is available for this search (which only happens // when it's anchored), then skip running a fallible DFA. The onepass @@ -831,17 +922,17 @@ impl Strategy for Core { // usually just wasted work. But, the lazy DFA is usually quite fast // and doesn't cost too much here. if self.onepass.get(&input).is_some() { - return self.search_slots_nofail(cache, &input, slots); + return self.try_search_slots_fallback(cache, &input, slots); } let m = match self.try_search_mayfail(cache, input) { Some(Ok(Some(m))) => m, - Some(Ok(None)) => return None, + Some(Ok(None)) => return Ok(None), Some(Err(_err)) => { trace!("fast capture search failed: {_err}"); - return self.search_slots_nofail(cache, input, slots); + return self.try_search_slots_fallback(cache, input, slots); } None => { - return self.search_slots_nofail(cache, input, slots); + return self.try_search_slots_fallback(cache, input, slots); } }; // At this point, now that we've found the bounds of the @@ -858,26 +949,26 @@ impl Strategy for Core { .clone() .span(m.start()..m.end()) .anchored(Anchored::Pattern(m.pattern())); - Some( - self.search_slots_nofail(cache, &input, slots) + Ok(Some( + self.try_search_slots_fallback(cache, &input, slots)? .expect("should find a match"), - ) + )) } #[cfg_attr(feature = "perf-inline", inline(always))] - fn which_overlapping_matches( + fn try_which_overlapping_matches( &self, cache: &mut Cache, input: &Input<'_>, patset: &mut PatternSet, - ) { + ) -> Result<(), RetryFailError> { if let Some(e) = self.dfa.get(input) { trace!( "using full DFA for overlapping search at {:?}", input.get_span() ); let _err = match e.try_which_overlapping_matches(input, patset) { - Ok(()) => return, + Ok(()) => return Ok(()), Err(err) => err, }; trace!("fast overlapping search failed: {_err}"); @@ -892,7 +983,7 @@ impl Strategy for Core { patset, ) { Ok(()) => { - return; + return Ok(()); } Err(err) => err, }; @@ -902,8 +993,12 @@ impl Strategy for Core { "using PikeVM for overlapping search at {:?}", input.get_span() ); - let e = self.pikevm.get(); - e.which_overlapping_matches(&mut cache.pikevm, input, patset) + let e = self + .pikevm + .get() + .ok_or_else(|| RetryFailError::from_offset(input.start()))?; + e.which_overlapping_matches(&mut cache.pikevm, input, patset); + Ok(()) } } @@ -1017,37 +1112,41 @@ impl Strategy for ReverseAnchored { } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option { + fn try_search( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result, RetryFailError> { if input.get_anchored().is_anchored() { - return self.core.search(cache, input); + return self.core.try_search(cache, input); } match self.try_search_half_anchored_rev(cache, input) { Err(_err) => { trace!("fast reverse anchored search failed: {_err}"); - self.core.search_nofail(cache, input) + self.core.try_search_fallback(cache, input) } - Ok(None) => None, + Ok(None) => Ok(None), Ok(Some(hm)) => { - Some(Match::new(hm.pattern(), hm.offset()..input.end())) + Ok(Some(Match::new(hm.pattern(), hm.offset()..input.end()))) } } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_half( + fn try_search_half( &self, cache: &mut Cache, input: &Input<'_>, - ) -> Option { + ) -> Result, RetryFailError> { if input.get_anchored().is_anchored() { - return self.core.search_half(cache, input); + return self.core.try_search_half(cache, input); } match self.try_search_half_anchored_rev(cache, input) { Err(_err) => { trace!("fast reverse anchored search failed: {_err}"); - self.core.search_half_nofail(cache, input) + self.core.try_search_half_fallback(cache, input) } - Ok(None) => None, + Ok(None) => Ok(None), Ok(Some(hm)) => { // Careful here! 'try_search_half' is a *forward* search that // only cares about the *end* position of a match. But @@ -1055,71 +1154,75 @@ impl Strategy for ReverseAnchored { // actually just throw that away here and, since we know we // have a match, return the only possible position at which a // match can occur: input.end(). - Some(HalfMatch::new(hm.pattern(), input.end())) + Ok(Some(HalfMatch::new(hm.pattern(), input.end()))) } } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + fn try_is_match( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result { if input.get_anchored().is_anchored() { - return self.core.is_match(cache, input); + return self.core.try_is_match(cache, input); } match self.try_search_half_anchored_rev(cache, input) { Err(_err) => { trace!("fast reverse anchored search failed: {_err}"); - self.core.is_match_nofail(cache, input) + self.core.try_is_match_fallback(cache, input) } - Ok(None) => false, - Ok(Some(_)) => true, + Ok(None) => Ok(false), + Ok(Some(_)) => Ok(true), } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_slots( + fn try_search_slots( &self, cache: &mut Cache, input: &Input<'_>, slots: &mut [Option], - ) -> Option { + ) -> Result, RetryFailError> { if input.get_anchored().is_anchored() { - return self.core.search_slots(cache, input, slots); + return self.core.try_search_slots(cache, input, slots); } match self.try_search_half_anchored_rev(cache, input) { Err(_err) => { trace!("fast reverse anchored search failed: {_err}"); - self.core.search_slots_nofail(cache, input, slots) + self.core.try_search_slots_fallback(cache, input, slots) } - Ok(None) => None, + Ok(None) => Ok(None), Ok(Some(hm)) => { if !self.core.is_capture_search_needed(slots.len()) { trace!("asked for slots unnecessarily, skipping captures"); let m = Match::new(hm.pattern(), hm.offset()..input.end()); copy_match_to_slots(m, slots); - return Some(m.pattern()); + return Ok(Some(m.pattern())); } let start = hm.offset(); let input = input .clone() .span(start..input.end()) .anchored(Anchored::Pattern(hm.pattern())); - self.core.search_slots_nofail(cache, &input, slots) + self.core.try_search_slots_fallback(cache, &input, slots) } } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn which_overlapping_matches( + fn try_which_overlapping_matches( &self, cache: &mut Cache, input: &Input<'_>, patset: &mut PatternSet, - ) { + ) -> Result<(), RetryFailError> { // It seems like this could probably benefit from a reverse anchored // optimization, perhaps by doing an overlapping reverse search (which // the DFAs do support). I haven't given it much thought though, and // I'm currently focus more on the single pattern case. - self.core.which_overlapping_matches(cache, input, patset) + self.core.try_which_overlapping_matches(cache, input, patset) } } @@ -1350,20 +1453,24 @@ impl Strategy for ReverseSuffix { } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option { + fn try_search( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result, RetryFailError> { if input.get_anchored().is_anchored() { - return self.core.search(cache, input); + return self.core.try_search(cache, input); } match self.try_search_half_start(cache, input) { Err(RetryError::Quadratic(_err)) => { trace!("reverse suffix optimization failed: {_err}"); - self.core.search(cache, input) + self.core.try_search(cache, input) } Err(RetryError::Fail(_err)) => { trace!("reverse suffix reverse fast search failed: {_err}"); - self.core.search_nofail(cache, input) + self.core.try_search_fallback(cache, input) } - Ok(None) => None, + Ok(None) => Ok(None), Ok(Some(hm_start)) => { let fwdinput = input .clone() @@ -1374,7 +1481,7 @@ impl Strategy for ReverseSuffix { trace!( "reverse suffix forward fast search failed: {_err}" ); - self.core.search_nofail(cache, input) + self.core.try_search_fallback(cache, input) } Ok(None) => { unreachable!( @@ -1382,36 +1489,36 @@ impl Strategy for ReverseSuffix { there must be a match", ) } - Ok(Some(hm_end)) => Some(Match::new( + Ok(Some(hm_end)) => Ok(Some(Match::new( hm_start.pattern(), hm_start.offset()..hm_end.offset(), - )), + ))), } } } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_half( + fn try_search_half( &self, cache: &mut Cache, input: &Input<'_>, - ) -> Option { + ) -> Result, RetryFailError> { if input.get_anchored().is_anchored() { - return self.core.search_half(cache, input); + return self.core.try_search_half(cache, input); } match self.try_search_half_start(cache, input) { Err(RetryError::Quadratic(_err)) => { trace!("reverse suffix half optimization failed: {_err}"); - self.core.search_half(cache, input) + self.core.try_search_half(cache, input) } Err(RetryError::Fail(_err)) => { trace!( "reverse suffix reverse fast half search failed: {_err}" ); - self.core.search_half_nofail(cache, input) + self.core.try_search_half_fallback(cache, input) } - Ok(None) => None, + Ok(None) => Ok(None), Ok(Some(hm_start)) => { // This is a bit subtle. It is tempting to just stop searching // at this point and return a half-match with an offset @@ -1431,7 +1538,7 @@ impl Strategy for ReverseSuffix { trace!( "reverse suffix forward fast search failed: {_err}" ); - self.core.search_half_nofail(cache, input) + self.core.try_search_half_fallback(cache, input) } Ok(None) => { unreachable!( @@ -1439,62 +1546,71 @@ impl Strategy for ReverseSuffix { there must be a match", ) } - Ok(Some(hm_end)) => Some(hm_end), + Ok(Some(hm_end)) => Ok(Some(hm_end)), } } } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + fn try_is_match( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result { if input.get_anchored().is_anchored() { - return self.core.is_match(cache, input); + return self.core.try_is_match(cache, input); } match self.try_search_half_start(cache, input) { Err(RetryError::Quadratic(_err)) => { trace!("reverse suffix half optimization failed: {_err}"); - self.core.is_match_nofail(cache, input) + self.core.try_is_match_fallback(cache, input) } Err(RetryError::Fail(_err)) => { trace!( "reverse suffix reverse fast half search failed: {_err}" ); - self.core.is_match_nofail(cache, input) + self.core.try_is_match_fallback(cache, input) } - Ok(None) => false, - Ok(Some(_)) => true, + Ok(None) => Ok(false), + Ok(Some(_)) => Ok(true), } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_slots( + fn try_search_slots( &self, cache: &mut Cache, input: &Input<'_>, slots: &mut [Option], - ) -> Option { + ) -> Result, RetryFailError> { if input.get_anchored().is_anchored() { - return self.core.search_slots(cache, input, slots); + return self.core.try_search_slots(cache, input, slots); } if !self.core.is_capture_search_needed(slots.len()) { trace!("asked for slots unnecessarily, trying fast path"); - let m = self.search(cache, input)?; + let m = match self.try_search(cache, input)? { + None => return Ok(None), + Some(m) => m, + }; copy_match_to_slots(m, slots); - return Some(m.pattern()); + return Ok(Some(m.pattern())); } let hm_start = match self.try_search_half_start(cache, input) { Err(RetryError::Quadratic(_err)) => { trace!("reverse suffix captures optimization failed: {_err}"); - return self.core.search_slots(cache, input, slots); + return self.core.try_search_slots(cache, input, slots); } Err(RetryError::Fail(_err)) => { trace!( "reverse suffix reverse fast captures search failed: \ {_err}" ); - return self.core.search_slots_nofail(cache, input, slots); + return self + .core + .try_search_slots_fallback(cache, input, slots); } - Ok(None) => return None, + Ok(None) => return Ok(None), Ok(Some(hm_start)) => hm_start, }; trace!( @@ -1508,17 +1624,17 @@ impl Strategy for ReverseSuffix { .clone() .span(start..input.end()) .anchored(Anchored::Pattern(hm_start.pattern())); - self.core.search_slots_nofail(cache, &input, slots) + self.core.try_search_slots_fallback(cache, &input, slots) } #[cfg_attr(feature = "perf-inline", inline(always))] - fn which_overlapping_matches( + fn try_which_overlapping_matches( &self, cache: &mut Cache, input: &Input<'_>, patset: &mut PatternSet, - ) { - self.core.which_overlapping_matches(cache, input, patset) + ) -> Result<(), RetryFailError> { + self.core.try_which_overlapping_matches(cache, input, patset) } } @@ -1810,91 +1926,104 @@ impl Strategy for ReverseInner { } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option { + fn try_search( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result, RetryFailError> { if input.get_anchored().is_anchored() { - return self.core.search(cache, input); + return self.core.try_search(cache, input); } match self.try_search_full(cache, input) { Err(RetryError::Quadratic(_err)) => { trace!("reverse inner optimization failed: {_err}"); - self.core.search(cache, input) + self.core.try_search(cache, input) } Err(RetryError::Fail(_err)) => { trace!("reverse inner fast search failed: {_err}"); - self.core.search_nofail(cache, input) + self.core.try_search_fallback(cache, input) } - Ok(matornot) => matornot, + Ok(matornot) => Ok(matornot), } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_half( + fn try_search_half( &self, cache: &mut Cache, input: &Input<'_>, - ) -> Option { + ) -> Result, RetryFailError> { if input.get_anchored().is_anchored() { - return self.core.search_half(cache, input); + return self.core.try_search_half(cache, input); } match self.try_search_full(cache, input) { Err(RetryError::Quadratic(_err)) => { trace!("reverse inner half optimization failed: {_err}"); - self.core.search_half(cache, input) + self.core.try_search_half(cache, input) } Err(RetryError::Fail(_err)) => { trace!("reverse inner fast half search failed: {_err}"); - self.core.search_half_nofail(cache, input) + self.core.try_search_half_fallback(cache, input) } - Ok(None) => None, - Ok(Some(m)) => Some(HalfMatch::new(m.pattern(), m.end())), + Ok(None) => Ok(None), + Ok(Some(m)) => Ok(Some(HalfMatch::new(m.pattern(), m.end()))), } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + fn try_is_match( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result { if input.get_anchored().is_anchored() { - return self.core.is_match(cache, input); + return self.core.try_is_match(cache, input); } match self.try_search_full(cache, input) { Err(RetryError::Quadratic(_err)) => { trace!("reverse inner half optimization failed: {_err}"); - self.core.is_match_nofail(cache, input) + self.core.try_is_match_fallback(cache, input) } Err(RetryError::Fail(_err)) => { trace!("reverse inner fast half search failed: {_err}"); - self.core.is_match_nofail(cache, input) + self.core.try_is_match_fallback(cache, input) } - Ok(None) => false, - Ok(Some(_)) => true, + Ok(None) => Ok(false), + Ok(Some(_)) => Ok(true), } } #[cfg_attr(feature = "perf-inline", inline(always))] - fn search_slots( + fn try_search_slots( &self, cache: &mut Cache, input: &Input<'_>, slots: &mut [Option], - ) -> Option { + ) -> Result, RetryFailError> { if input.get_anchored().is_anchored() { - return self.core.search_slots(cache, input, slots); + return self.core.try_search_slots(cache, input, slots); } if !self.core.is_capture_search_needed(slots.len()) { trace!("asked for slots unnecessarily, trying fast path"); - let m = self.search(cache, input)?; + let m = match self.try_search(cache, input)? { + None => return Ok(None), + Some(m) => m, + }; copy_match_to_slots(m, slots); - return Some(m.pattern()); + return Ok(Some(m.pattern())); } let m = match self.try_search_full(cache, input) { Err(RetryError::Quadratic(_err)) => { trace!("reverse inner captures optimization failed: {_err}"); - return self.core.search_slots(cache, input, slots); + return self.core.try_search_slots(cache, input, slots); } Err(RetryError::Fail(_err)) => { trace!("reverse inner fast captures search failed: {_err}"); - return self.core.search_slots_nofail(cache, input, slots); + return self + .core + .try_search_slots_fallback(cache, input, slots); } - Ok(None) => return None, + Ok(None) => return Ok(None), Ok(Some(m)) => m, }; trace!( @@ -1907,17 +2036,17 @@ impl Strategy for ReverseInner { .clone() .span(m.start()..m.end()) .anchored(Anchored::Pattern(m.pattern())); - self.core.search_slots_nofail(cache, &input, slots) + self.core.try_search_slots_fallback(cache, &input, slots) } #[cfg_attr(feature = "perf-inline", inline(always))] - fn which_overlapping_matches( + fn try_which_overlapping_matches( &self, cache: &mut Cache, input: &Input<'_>, patset: &mut PatternSet, - ) { - self.core.which_overlapping_matches(cache, input, patset) + ) -> Result<(), RetryFailError> { + self.core.try_which_overlapping_matches(cache, input, patset) } } diff --git a/regex-automata/src/meta/wrappers.rs b/regex-automata/src/meta/wrappers.rs index 8d6f738e4c..b61c4f846d 100644 --- a/regex-automata/src/meta/wrappers.rs +++ b/regex-automata/src/meta/wrappers.rs @@ -46,7 +46,7 @@ use crate::hybrid; use crate::nfa::thompson::backtrack; #[derive(Debug)] -pub(crate) struct PikeVM(PikeVMEngine); +pub(crate) struct PikeVM(Option); impl PikeVM { pub(crate) fn new( @@ -54,7 +54,11 @@ impl PikeVM { pre: Option, nfa: &NFA, ) -> Result { - PikeVMEngine::new(info, pre, nfa).map(PikeVM) + if !info.config().get_pikevm() { + debug!("PikeVM disabled by config"); + return Ok(PikeVM(None)); + } + PikeVMEngine::new(info, pre, nfa).map(|e| PikeVM(Some(e))) } pub(crate) fn create_cache(&self) -> PikeVMCache { @@ -62,8 +66,8 @@ impl PikeVM { } #[cfg_attr(feature = "perf-inline", inline(always))] - pub(crate) fn get(&self) -> &PikeVMEngine { - &self.0 + pub(crate) fn get(&self) -> Option<&PikeVMEngine> { + self.0.as_ref() } } @@ -126,7 +130,9 @@ impl PikeVMCache { } pub(crate) fn reset(&mut self, builder: &PikeVM) { - self.get(&builder.get().0).reset(&builder.get().0); + if let Some(e) = builder.get() { + self.get(&e.0).reset(&e.0); + } } pub(crate) fn memory_usage(&self) -> usize { @@ -139,7 +145,11 @@ impl PikeVMCache { } #[derive(Debug)] -pub(crate) struct BoundedBacktracker(Option); +pub(crate) struct BoundedBacktracker( + Option, + // Whether the PikeVM is available to defer to. + bool, +); impl BoundedBacktracker { pub(crate) fn new( @@ -147,7 +157,8 @@ impl BoundedBacktracker { pre: Option, nfa: &NFA, ) -> Result { - BoundedBacktrackerEngine::new(info, pre, nfa).map(BoundedBacktracker) + let engine = BoundedBacktrackerEngine::new(info, pre, nfa)?; + Ok(BoundedBacktracker(engine, info.config().get_pikevm())) } pub(crate) fn create_cache(&self) -> BoundedBacktrackerCache { @@ -172,7 +183,11 @@ impl BoundedBacktracker { // Now, if the haystack is really short already, then we allow the // backtracker to run. (This hasn't been litigated quantitatively with // benchmarks. Just a hunch.) - if input.get_earliest() && input.haystack().len() > 128 { + // + // This only holds while there is another engine to wait for. With the + // PikeVM disabled the backtracker is the last one standing, so running + // it beats giving up. + if self.1 && input.get_earliest() && input.haystack().len() > 128 { return None; } // If the backtracker is just going to return an error because the @@ -203,7 +218,12 @@ impl BoundedBacktrackerEngine { { return Ok(None); } - let backtrack_config = backtrack::Config::new().prefilter(pre); + let mut backtrack_config = backtrack::Config::new().prefilter(pre); + if let Some(capacity) = + info.config().get_backtrack_visited_capacity() + { + backtrack_config = backtrack_config.visited_capacity(capacity); + } let engine = backtrack::Builder::new() .configure(backtrack_config) .build_from_nfa(nfa.clone()) diff --git a/regex-automata/src/util/search.rs b/regex-automata/src/util/search.rs index 37999f4bba..d1ac35f9a9 100644 --- a/regex-automata/src/util/search.rs +++ b/regex-automata/src/util/search.rs @@ -1820,6 +1820,12 @@ impl MatchError { MatchError::new(MatchErrorKind::GaveUp { offset }) } + /// Create a new "no engine" error. The given `offset` corresponds to the + /// place in the haystack at which the search stopped. + pub fn no_engine(offset: usize) -> MatchError { + MatchError::new(MatchErrorKind::NoEngine { offset }) + } + /// Create a new "haystack too long" error. The given `len` corresponds to /// the length of the haystack that was problematic. /// @@ -1876,6 +1882,21 @@ pub enum MatchErrorKind { /// The length of the haystack that exceeded the limit. len: usize, }, + /// The search stopped because no regex engine was available to continue + /// it. + /// + /// This can only occur in the meta regex engine, and only when the PikeVM + /// has been disabled via + /// [`meta::Config::pikevm`](crate::meta::Config::pikevm). The PikeVM is + /// the meta regex engine's only engine that can handle every regex on + /// every haystack, so without it, a search that the other engines quit + /// on or gave up on has nothing left to run it. (It will not return this + /// error by default.) + NoEngine { + /// The offset at which the search stopped. This corresponds to the + /// position immediately following the last byte scanned. + offset: usize, + }, /// An error indicating that a particular type of anchored search was /// requested, but that the regex engine does not support it. /// @@ -1916,6 +1937,9 @@ impl core::fmt::Display for MatchError { MatchErrorKind::HaystackTooLong { len } => { write!(f, "haystack of length {len} is too long") } + MatchErrorKind::NoEngine { offset } => { + write!(f, "no regex engine available at offset {offset}") + } MatchErrorKind::UnsupportedAnchored { mode: Anchored::Yes } => { write!(f, "anchored searches are not supported or enabled") }