automata: add targeted inline(always) annotations

It turns out there are quite a few routines that don't seem to get
inlined and we need to help the compiler make the right choice. This
ends up having a pretty dramatic impact on benchmarks dominated by
latency.

The annotations are of course gated on the `perf-inline` feature.

Fixes #1029

Author: BurntSushi

regex-automata/src/dfa/dense.rs

@@ -3012,6 +3012,7 @@ impl<T: AsRef<[u32]>> DFA<T> {
     /// Returns the index of the match state for the given ID. If the
     /// given ID does not correspond to a match state, then this may
     /// panic or produce an incorrect result.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     fn match_state_index(&self, id: StateID) -> usize {
         debug_assert!(self.is_match_state(id));
         // This is one of the places where we rely on the fact that match
@@ -4599,6 +4600,7 @@ impl<T: AsRef<[u32]>> MatchStates<T> {
     ///
     /// The match index is the index of the pattern ID for the given state.
     /// The index must be less than `self.pattern_len(state_index)`.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     fn pattern_id(&self, state_index: usize, match_index: usize) -> PatternID {
         self.pattern_id_slice(state_index)[match_index]
     }
@@ -4607,6 +4609,7 @@ impl<T: AsRef<[u32]>> MatchStates<T> {
     ///
     /// The match state index is the state index minus the state index of the
     /// first match state in the DFA.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     fn pattern_len(&self, state_index: usize) -> usize {
         self.slices()[state_index * 2 + 1].as_usize()
     }
@@ -4615,24 +4618,28 @@ impl<T: AsRef<[u32]>> MatchStates<T> {
     ///
     /// The match state index is the state index minus the state index of the
     /// first match state in the DFA.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     fn pattern_id_slice(&self, state_index: usize) -> &[PatternID] {
         let start = self.slices()[state_index * 2].as_usize();
         let len = self.pattern_len(state_index);
         &self.pattern_ids()[start..start + len]
     }
 
     /// Returns the pattern ID offset slice of u32 as a slice of PatternID.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     fn slices(&self) -> &[PatternID] {
         wire::u32s_to_pattern_ids(self.slices.as_ref())
     }
 
     /// Returns the total number of match states.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     fn len(&self) -> usize {
         assert_eq!(0, self.slices().len() % 2);
         self.slices().len() / 2
     }
 
     /// Returns the pattern ID slice of u32 as a slice of PatternID.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     fn pattern_ids(&self) -> &[PatternID] {
         wire::u32s_to_pattern_ids(self.pattern_ids.as_ref())
     }

regex-automata/src/dfa/start.rs

@@ -62,10 +62,12 @@ impl StartKind {
         size_of::<u32>()
     }
 
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     pub(crate) fn has_unanchored(&self) -> bool {
         matches!(*self, StartKind::Both | StartKind::Unanchored)
     }
 
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     pub(crate) fn has_anchored(&self) -> bool {
         matches!(*self, StartKind::Both | StartKind::Anchored)
     }

regex-automata/src/util/alphabet.rs

@@ -796,6 +796,7 @@ impl ByteSet {
     }
 
     /// Return true if and only if this set is empty.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     pub(crate) fn is_empty(&self) -> bool {
         self.bits.0 == [0, 0]
     }

regex-automata/src/util/search.rs

@@ -1526,6 +1526,7 @@ impl Anchored {
     /// assert!(Anchored::Yes.is_anchored());
     /// assert!(Anchored::Pattern(PatternID::ZERO).is_anchored());
     /// ```
+    #[inline]
     pub fn is_anchored(&self) -> bool {
         matches!(*self, Anchored::Yes | Anchored::Pattern(_))
     }
@@ -1544,6 +1545,7 @@ impl Anchored {
     /// let pid = PatternID::must(5);
     /// assert_eq!(Some(pid), Anchored::Pattern(pid).pattern());
     /// ```
+    #[inline]
     pub fn pattern(&self) -> Option<PatternID> {
         match *self {
             Anchored::Pattern(pid) => Some(pid),

regex-automata/src/util/start.rs

@@ -72,6 +72,7 @@ impl StartByteMap {
     }
 
     /// Return the forward starting configuration for the given `input`.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     pub(crate) fn fwd(&self, input: &Input) -> Start {
         match input
             .start()
@@ -84,13 +85,15 @@ impl StartByteMap {
     }
 
     /// Return the reverse starting configuration for the given `input`.
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     pub(crate) fn rev(&self, input: &Input) -> Start {
         match input.haystack().get(input.end()) {
             None => Start::Text,
             Some(&byte) => self.get(byte),
         }
     }
 
+    #[cfg_attr(feature = "perf-inline", inline(always))]
     fn get(&self, byte: u8) -> Start {
         self.map[usize::from(byte)]
     }

regex-automata/src/util/wire.rs

@@ -270,6 +270,7 @@ impl core::fmt::Display for DeserializeError {
 }
 
 /// Safely converts a `&[u32]` to `&[StateID]` with zero cost.
+#[cfg_attr(feature = "perf-inline", inline(always))]
 pub(crate) fn u32s_to_state_ids(slice: &[u32]) -> &[StateID] {
     // SAFETY: This is safe because StateID is defined to have the same memory
     // representation as a u32 (it is repr(transparent)). While not every u32
@@ -300,6 +301,7 @@ pub(crate) fn u32s_to_state_ids_mut(slice: &mut [u32]) -> &mut [StateID] {
 }
 
 /// Safely converts a `&[u32]` to `&[PatternID]` with zero cost.
+#[cfg_attr(feature = "perf-inline", inline(always))]
 pub(crate) fn u32s_to_pattern_ids(slice: &[u32]) -> &[PatternID] {
     // SAFETY: This is safe because PatternID is defined to have the same
     // memory representation as a u32 (it is repr(transparent)). While not