syntax: small HIR simplifications

This makes it so 'a{1}' is rewritten as 'a' and '[a]' is rewritten as
'a'.

A lot of the tests expected '[a]' to get preserved as a class in the
HIR, so this required a bit of surgery.

Author: BurntSushi

regex-syntax/src/hir/mod.rs

@@ -122,16 +122,16 @@ impl core::fmt::Display for ErrorKind {
 
 /// A high-level intermediate representation (HIR) for a regular expression.
 ///
-/// The HIR of a regular expression represents an intermediate step between its
-/// abstract syntax (a structured description of the concrete syntax) and
-/// compiled byte codes. The purpose of HIR is to make regular expressions
+/// The HIR of a regular expression represents an intermediate step between
+/// its abstract syntax (a structured description of the concrete syntax) and
+/// an actual regex matcher. The purpose of HIR is to make regular expressions
 /// easier to analyze. In particular, the AST is much more complex than the
 /// HIR. For example, while an AST supports arbitrarily nested character
 /// classes, the HIR will flatten all nested classes into a single set. The HIR
 /// will also "compile away" every flag present in the concrete syntax. For
 /// example, users of HIR expressions never need to worry about case folding;
-/// it is handled automatically by the translator (e.g., by translating `(?i)A`
-/// to `[aA]`).
+/// it is handled automatically by the translator (e.g., by translating
+/// `(?i:A)` to `[aA]`).
 ///
 /// If the HIR was produced by a translator that disallows invalid UTF-8, then
 /// the HIR is guaranteed to match UTF-8 exclusively.
@@ -150,11 +150,13 @@ impl core::fmt::Display for ErrorKind {
 /// 2. Every HIR expression contains attributes that are defined inductively,
 ///    and can be computed cheaply during the construction process. For
 ///    example, one such attribute is whether the expression must match at the
-///    beginning of the text.
+///    beginning of the haystack.
 ///
 /// Also, an `Hir`'s `fmt::Display` implementation prints an HIR as a regular
 /// expression pattern string, and uses constant stack space and heap space
-/// proportional to the size of the `Hir`.
+/// proportional to the size of the `Hir`. The regex it prints is guaranteed to
+/// be _semantically_ equivalent to the original concrete syntax, but it may
+/// look very different. (And potentially not practically readable by a human.)
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Hir {
     /// The underlying HIR kind.
@@ -252,6 +254,9 @@ impl Hir {
 
     /// Creates a class HIR expression.
     pub fn class(class: Class) -> Hir {
+        if let Some(bytes) = class.literal() {
+            return Hir::literal(bytes);
+        }
         let props = Properties::class(&class);
         Hir { kind: HirKind::Class(class), props }
     }
@@ -267,8 +272,12 @@ impl Hir {
         // The regex 'a{0}' is always equivalent to the empty regex. This is
         // true even when 'a' is an expression that never matches anything
         // (like '\P{any}').
+        //
+        // Additionally, the regex 'a{1}' is always equivalent to 'a'.
         if rep.min == 0 && rep.max == Some(0) {
             return Hir::empty();
+        } else if rep.min == 1 && rep.max == Some(1) {
+            return *rep.hir;
         }
         let props = Properties::repetition(&rep);
         Hir { kind: HirKind::Repetition(rep), props }
@@ -541,6 +550,18 @@ impl Class {
             Class::Bytes(ref x) => x.maximum_len(),
         }
     }
+
+    /// If this class consists of exactly one element (whether a codepoint or a
+    /// byte), then return it as a literal byte string.
+    ///
+    /// If this class is empty or contains more than one element, then `None`
+    /// is returned.
+    pub fn literal(&self) -> Option<Vec<u8>> {
+        match *self {
+            Class::Unicode(ref x) => x.literal(),
+            Class::Bytes(ref x) => x.literal(),
+        }
+    }
 }
 
 /// A set of characters represented by Unicode scalar values.
@@ -680,6 +701,20 @@ impl ClassUnicode {
         // Correct because c1 < c2 implies c1.len_utf8() < c2.len_utf8().
         Some(last.end.len_utf8())
     }
+
+    /// If this class consists of exactly one codepoint, then return it as
+    /// a literal byte string.
+    ///
+    /// If this class is empty or contains more than one codepoint, then `None`
+    /// is returned.
+    pub fn literal(&self) -> Option<Vec<u8>> {
+        let rs = self.ranges();
+        if rs.len() == 1 && rs[0].start == rs[0].end {
+            Some(rs[0].start.encode_utf8(&mut [0; 4]).to_string().into_bytes())
+        } else {
+            None
+        }
+    }
 }
 
 /// An iterator over all ranges in a Unicode character class.
@@ -932,6 +967,20 @@ impl ClassBytes {
             Some(1)
         }
     }
+
+    /// If this class consists of exactly one byte, then return it as
+    /// a literal byte string.
+    ///
+    /// If this class is empty or contains more than one byte, then `None`
+    /// is returned.
+    pub fn literal(&self) -> Option<Vec<u8>> {
+        let rs = self.ranges();
+        if rs.len() == 1 && rs[0].start == rs[0].end {
+            Some(vec![rs[0].start])
+        } else {
+            None
+        }
+    }
 }
 
 /// An iterator over all ranges in a byte character class.

regex-syntax/src/hir/print.rs

@@ -129,6 +129,11 @@ impl<W: fmt::Write> Visitor for Writer<W> {
                 for range in cls.iter() {
                     if range.start() == range.end() {
                         self.write_literal_char(range.start())?;
+                    } else if u32::from(range.start()) + 1
+                        == u32::from(range.end())
+                    {
+                        self.write_literal_char(range.start())?;
+                        self.write_literal_char(range.end())?;
                     } else {
                         self.write_literal_char(range.start())?;
                         self.wtr.write_str("-")?;
@@ -142,6 +147,9 @@ impl<W: fmt::Write> Visitor for Writer<W> {
                 for range in cls.iter() {
                     if range.start() == range.end() {
                         self.write_literal_class_byte(range.start())?;
+                    } else if range.start() + 1 == range.end() {
+                        self.write_literal_class_byte(range.start())?;
+                        self.write_literal_class_byte(range.end())?;
                     } else {
                         self.write_literal_class_byte(range.start())?;
                         self.wtr.write_str("-")?;
@@ -327,26 +335,28 @@ mod tests {
 
     #[test]
     fn print_class() {
-        roundtrip(r"[a]", r"[a]");
+        roundtrip(r"[a]", r"a");
+        roundtrip(r"[ab]", r"[ab]");
         roundtrip(r"[a-z]", r"[a-z]");
         roundtrip(r"[a-z--b-c--x-y]", r"[ad-wz]");
-        roundtrip(r"[^\x01-\u{10FFFF}]", "[\u{0}]");
-        roundtrip(r"[-]", r"[\-]");
+        roundtrip(r"[^\x01-\u{10FFFF}]", "\u{0}");
+        roundtrip(r"[-]", r"\-");
         roundtrip(r"[☃-⛄]", r"[☃-⛄]");
 
-        roundtrip(r"(?-u)[a]", r"(?-u:[a])");
+        roundtrip(r"(?-u)[a]", r"a");
+        roundtrip(r"(?-u)[ab]", r"(?-u:[ab])");
         roundtrip(r"(?-u)[a-z]", r"(?-u:[a-z])");
         roundtrip_bytes(r"(?-u)[a-\xFF]", r"(?-u:[a-\xFF])");
 
         // The following test that the printer escapes meta characters
         // in character classes.
-        roundtrip(r"[\[]", r"[\[]");
+        roundtrip(r"[\[]", r"\[");
         roundtrip(r"[Z-_]", r"[Z-_]");
         roundtrip(r"[Z-_--Z]", r"[\[-_]");
 
         // The following test that the printer escapes meta characters
         // in byte oriented character classes.
-        roundtrip_bytes(r"(?-u)[\[]", r"(?-u:[\[])");
+        roundtrip_bytes(r"(?-u)[\[]", r"\[");
         roundtrip_bytes(r"(?-u)[Z-_]", r"(?-u:[Z-_])");
         roundtrip_bytes(r"(?-u)[Z-_--Z]", r"(?-u:[\[-_])");
     }

regex-syntax/src/hir/translate.rs

@@ -1428,19 +1428,11 @@ mod tests {
     }
 
     fn hir_uclass(ranges: &[(char, char)]) -> Hir {
-        let ranges: Vec<hir::ClassUnicodeRange> = ranges
-            .iter()
-            .map(|&(s, e)| hir::ClassUnicodeRange::new(s, e))
-            .collect();
-        Hir::class(hir::Class::Unicode(hir::ClassUnicode::new(ranges)))
+        Hir::class(uclass(ranges))
     }
 
     fn hir_bclass(ranges: &[(u8, u8)]) -> Hir {
-        let ranges: Vec<hir::ClassBytesRange> = ranges
-            .iter()
-            .map(|&(s, e)| hir::ClassBytesRange::new(s, e))
-            .collect();
-        Hir::class(hir::Class::Bytes(hir::ClassBytes::new(ranges)))
+        Hir::class(bclass(ranges))
     }
 
     fn hir_case_fold(expr: Hir) -> Hir {
@@ -1463,6 +1455,33 @@ mod tests {
         }
     }
 
+    fn uclass(ranges: &[(char, char)]) -> hir::Class {
+        let ranges: Vec<hir::ClassUnicodeRange> = ranges
+            .iter()
+            .map(|&(s, e)| hir::ClassUnicodeRange::new(s, e))
+            .collect();
+        hir::Class::Unicode(hir::ClassUnicode::new(ranges))
+    }
+
+    fn bclass(ranges: &[(u8, u8)]) -> hir::Class {
+        let ranges: Vec<hir::ClassBytesRange> = ranges
+            .iter()
+            .map(|&(s, e)| hir::ClassBytesRange::new(s, e))
+            .collect();
+        hir::Class::Bytes(hir::ClassBytes::new(ranges))
+    }
+
+    #[cfg(feature = "unicode-case")]
+    fn class_case_fold(mut cls: hir::Class) -> Hir {
+        cls.case_fold_simple();
+        Hir::class(cls)
+    }
+
+    fn class_negate(mut cls: hir::Class) -> Hir {
+        cls.negate();
+        Hir::class(cls)
+    }
+
     #[allow(dead_code)]
     fn hir_union(expr1: Hir, expr2: Hir) -> Hir {
         use crate::hir::Class::{Bytes, Unicode};
@@ -2522,8 +2541,9 @@ mod tests {
 
     #[test]
     fn class_bracketed() {
-        assert_eq!(t("[a]"), hir_uclass(&[('a', 'a')]));
-        assert_eq!(t("[^[a]]"), hir_negate(hir_uclass(&[('a', 'a')])));
+        assert_eq!(t("[a]"), hir_lit("a"));
+        assert_eq!(t("[ab]"), hir_uclass(&[('a', 'b')]));
+        assert_eq!(t("[^[a]]"), class_negate(uclass(&[('a', 'a')])));
         assert_eq!(t("[a-z]"), hir_uclass(&[('a', 'z')]));
         assert_eq!(t("[a-fd-h]"), hir_uclass(&[('a', 'h')]));
         assert_eq!(t("[a-fg-m]"), hir_uclass(&[('a', 'm')]));
@@ -2586,11 +2606,11 @@ mod tests {
         );
         assert_eq!(t("(?i-u)[k]"), hir_bclass(&[(b'K', b'K'), (b'k', b'k'),]));
 
-        assert_eq!(t("[^a]"), hir_negate(hir_uclass(&[('a', 'a')])));
-        assert_eq!(t(r"[^\x00]"), hir_negate(hir_uclass(&[('\0', '\0')])));
+        assert_eq!(t("[^a]"), class_negate(uclass(&[('a', 'a')])));
+        assert_eq!(t(r"[^\x00]"), class_negate(uclass(&[('\0', '\0')])));
         assert_eq!(
             t_bytes("(?-u)[^a]"),
-            hir_negate(hir_bclass(&[(b'a', b'a')]))
+            class_negate(bclass(&[(b'a', b'a')]))
         );
         #[cfg(any(feature = "unicode-perl", feature = "unicode-gencat"))]
         assert_eq!(
@@ -2778,8 +2798,8 @@ mod tests {
 
     #[test]
     fn class_bracketed_nested() {
-        assert_eq!(t(r"[a[^c]]"), hir_negate(hir_uclass(&[('c', 'c')])));
-        assert_eq!(t(r"[a-b[^c]]"), hir_negate(hir_uclass(&[('c', 'c')])));
+        assert_eq!(t(r"[a[^c]]"), class_negate(uclass(&[('c', 'c')])));
+        assert_eq!(t(r"[a-b[^c]]"), class_negate(uclass(&[('c', 'c')])));
         assert_eq!(t(r"[a-c[^c]]"), hir_negate(hir_uclass(&[])));
 
         assert_eq!(t(r"[^a[^c]]"), hir_uclass(&[('c', 'c')]));
@@ -2788,12 +2808,12 @@ mod tests {
         #[cfg(feature = "unicode-case")]
         assert_eq!(
             t(r"(?i)[a[^c]]"),
-            hir_negate(hir_case_fold(hir_uclass(&[('c', 'c')])))
+            hir_negate(class_case_fold(uclass(&[('c', 'c')])))
         );
         #[cfg(feature = "unicode-case")]
         assert_eq!(
             t(r"(?i)[a-b[^c]]"),
-            hir_negate(hir_case_fold(hir_uclass(&[('c', 'c')])))
+            hir_negate(class_case_fold(uclass(&[('c', 'c')])))
         );
 
         #[cfg(feature = "unicode-case")]
@@ -3239,6 +3259,10 @@ mod tests {
         assert!(props(r"ab").is_literal());
         assert!(props(r"abc").is_literal());
         assert!(props(r"(?m)abc").is_literal());
+        assert!(props(r"(?:a)").is_literal());
+        assert!(props(r"foo(?:a)").is_literal());
+        assert!(props(r"(?:a)foo").is_literal());
+        assert!(props(r"[a]").is_literal());
 
         // Negative examples.
         assert!(!props(r"").is_literal());
@@ -3248,7 +3272,7 @@ mod tests {
         assert!(!props(r"a+").is_literal());
         assert!(!props(r"foo(a)").is_literal());
         assert!(!props(r"(a)foo").is_literal());
-        assert!(!props(r"[a]").is_literal());
+        assert!(!props(r"[ab]").is_literal());
     }
 
     #[test]
@@ -3262,6 +3286,11 @@ mod tests {
         assert!(props(r"a|b|c").is_alternation_literal());
         assert!(props(r"foo|bar").is_alternation_literal());
         assert!(props(r"foo|bar|baz").is_alternation_literal());
+        assert!(props(r"[a]").is_alternation_literal());
+        assert!(props(r"[a]|b").is_alternation_literal());
+        assert!(props(r"a|[b]").is_alternation_literal());
+        assert!(props(r"(?:a)|b").is_alternation_literal());
+        assert!(props(r"a|(?:b)").is_alternation_literal());
 
         // Negative examples.
         assert!(!props(r"").is_alternation_literal());
@@ -3270,9 +3299,9 @@ mod tests {
         assert!(!props(r"a+").is_alternation_literal());
         assert!(!props(r"foo(a)").is_alternation_literal());
         assert!(!props(r"(a)foo").is_alternation_literal());
-        assert!(!props(r"[a]").is_alternation_literal());
-        assert!(!props(r"[a]|b").is_alternation_literal());
-        assert!(!props(r"a|[b]").is_alternation_literal());
+        assert!(!props(r"[ab]").is_alternation_literal());
+        assert!(!props(r"[ab]|b").is_alternation_literal());
+        assert!(!props(r"a|[ab]").is_alternation_literal());
         assert!(!props(r"(a)|b").is_alternation_literal());
         assert!(!props(r"a|(b)").is_alternation_literal());
     }