feat!: Pattern parser in is now stateful to allow options for how to parse ignore patterns.

That way it can support settings and other state that affect parsing.
This affects various crates which are all marked as breaking now.

Author: Byron

gix-attributes/src/search/attributes.rs

@@ -58,7 +58,8 @@ impl Search {
     ) -> std::io::Result<bool> {
         // TODO: should `Pattern` trait use an instance as first argument to carry this information
         //       (so no `retain` later, it's slower than skipping)
-        let was_added = gix_glob::search::add_patterns_file(&mut self.patterns, source, follow_symlinks, root, buf)?;
+        let was_added =
+            gix_glob::search::add_patterns_file(&mut self.patterns, source, follow_symlinks, root, buf, Attributes)?;
         if was_added {
             let last = self.patterns.last_mut().expect("just added");
             if !allow_macros {
@@ -80,7 +81,8 @@ impl Search {
         collection: &mut MetadataCollection,
         allow_macros: bool,
     ) {
-        self.patterns.push(pattern::List::from_bytes(bytes, source, root));
+        self.patterns
+            .push(pattern::List::from_bytes(bytes, source, root, Attributes));
         let last = self.patterns.last_mut().expect("just added");
         if !allow_macros {
             last.patterns
@@ -124,7 +126,7 @@ impl Search {
 impl Pattern for Attributes {
     type Value = Value;
 
-    fn bytes_to_patterns(bytes: &[u8], _source: &std::path::Path) -> Vec<pattern::Mapping<Self::Value>> {
+    fn bytes_to_patterns(&self, bytes: &[u8], _source: &std::path::Path) -> Vec<pattern::Mapping<Self::Value>> {
         fn into_owned_assignments<'a>(
             attrs: impl Iterator<Item = Result<crate::AssignmentRef<'a>, crate::name::Error>>,
         ) -> Option<Assignments> {

gix-glob/src/search/mod.rs

@@ -15,20 +15,28 @@ pub trait Pattern: Clone + PartialEq + Eq + std::fmt::Debug + std::hash::Hash +
     type Value: PartialEq + Eq + std::fmt::Debug + std::hash::Hash + Ord + PartialOrd + Clone;
 
     /// Parse all patterns in `bytes` line by line, ignoring lines with errors, and collect them.
-    fn bytes_to_patterns(bytes: &[u8], source: &Path) -> Vec<pattern::Mapping<Self::Value>>;
+    fn bytes_to_patterns(&self, bytes: &[u8], source: &Path) -> Vec<pattern::Mapping<Self::Value>>;
 }
 
 /// Add the given file at `source` if it exists, otherwise do nothing.
 /// If a `root` is provided, it's not considered a global file anymore.
+/// `parse` is a way to parse bytes to pattern.
 /// Returns `true` if the file was added, or `false` if it didn't exist.
 pub fn add_patterns_file<T: Pattern>(
     patterns: &mut Vec<pattern::List<T>>,
     source: PathBuf,
     follow_symlinks: bool,
     root: Option<&Path>,
     buf: &mut Vec<u8>,
+    parse: T,
 ) -> std::io::Result<bool> {
     let previous_len = patterns.len();
-    patterns.extend(pattern::List::<T>::from_file(source, root, follow_symlinks, buf)?);
+    patterns.extend(pattern::List::<T>::from_file(
+        source,
+        root,
+        follow_symlinks,
+        buf,
+        parse,
+    )?);
     Ok(patterns.len() != previous_len)
 }

gix-glob/src/search/pattern.rs

@@ -79,8 +79,9 @@ where
     /// `source_file` is the location of the `bytes` which represents a list of patterns, one pattern per line.
     /// If `root` is `Some(…)` it's used to see `source_file` as relative to itself, if `source_file` is absolute.
     /// If source is relative and should be treated as base, set `root` to `Some("")`.
-    pub fn from_bytes(bytes: &[u8], source_file: PathBuf, root: Option<&Path>) -> Self {
-        let patterns = T::bytes_to_patterns(bytes, source_file.as_path());
+    /// `parse` is a way to parse bytes to pattern.
+    pub fn from_bytes(bytes: &[u8], source_file: PathBuf, root: Option<&Path>, parse: T) -> Self {
+        let patterns = parse.bytes_to_patterns(bytes, source_file.as_path());
         let base = root
             .and_then(|root| source_file.parent().expect("file").strip_prefix(root).ok())
             .and_then(|base| {
@@ -101,14 +102,17 @@ where
 
     /// Create a pattern list from the `source` file, which may be located underneath `root`, while optionally
     /// following symlinks with `follow_symlinks`, providing `buf` to temporarily store the data contained in the file.
+    /// `parse` is a way to parse bytes to pattern.
     pub fn from_file(
         source: impl Into<PathBuf>,
         root: Option<&Path>,
         follow_symlinks: bool,
         buf: &mut Vec<u8>,
+        parse: T,
     ) -> std::io::Result<Option<Self>> {
         let source = source.into();
-        Ok(read_in_full_ignore_missing(&source, follow_symlinks, buf)?.then(|| Self::from_bytes(buf, source, root)))
+        Ok(read_in_full_ignore_missing(&source, follow_symlinks, buf)?
+            .then(|| Self::from_bytes(buf, source, root, parse)))
     }
 }
 

gix-glob/tests/search/pattern.rs

@@ -15,15 +15,15 @@ mod list {
     impl Pattern for Dummy {
         type Value = ();
 
-        fn bytes_to_patterns(_bytes: &[u8], _source: &Path) -> Vec<Mapping<Self::Value>> {
+        fn bytes_to_patterns(&self, _bytes: &[u8], _source: &Path) -> Vec<Mapping<Self::Value>> {
             vec![]
         }
     }
 
     #[test]
     fn from_bytes_base() {
         {
-            let list = List::<Dummy>::from_bytes(&[], "a/b/source".into(), None);
+            let list = List::from_bytes(&[], "a/b/source".into(), None, Dummy);
             assert_eq!(list.base, None, "no root always means no-base, i.e. globals lists");
             assert_eq!(
                 list.source.as_deref(),
@@ -34,7 +34,7 @@ mod list {
 
         {
             let cwd = std::env::current_dir().expect("cwd available");
-            let list = List::<Dummy>::from_bytes(&[], cwd.join("a/b/source"), Some(cwd.as_path()));
+            let list = List::from_bytes(&[], cwd.join("a/b/source"), Some(cwd.as_path()), Dummy);
             assert_eq!(
                 list.base.as_ref().expect("set"),
                 "a/b/",
@@ -48,7 +48,7 @@ mod list {
         }
 
         {
-            let list = List::<Dummy>::from_bytes(&[], "a/b/source".into(), Some(Path::new("c/")));
+            let list = List::from_bytes(&[], "a/b/source".into(), Some(Path::new("c/")), Dummy);
             assert_eq!(
                 list.base, None,
                 "if root doesn't contain source, it silently skips it as base"
@@ -63,7 +63,7 @@ mod list {
 
     #[test]
     fn strip_base_handle_recompute_basename_pos() {
-        let list = List::<Dummy>::from_bytes(&[], "a/b/source".into(), Some(Path::new("")));
+        let list = List::from_bytes(&[], "a/b/source".into(), Some(Path::new("")), Dummy);
         assert_eq!(
             list.base.as_ref().expect("set"),
             "a/b/",
@@ -91,7 +91,7 @@ mod list {
             Path::new(".").join("non-existing-dir").join("pattern-file"),
             Path::new("file").to_owned(),
         ] {
-            let list = List::<Dummy>::from_file(path, None, false, &mut buf).expect("no io error");
+            let list = List::from_file(path, None, false, &mut buf, Dummy).expect("no io error");
             assert!(list.is_none(), "the file does not exist");
         }
     }
@@ -102,7 +102,7 @@ mod list {
         let dir_path = tmp.path().join(".gitignore");
         std::fs::create_dir(&dir_path)?;
         let mut buf = Vec::new();
-        let list = List::<Dummy>::from_file(dir_path, None, false, &mut buf).expect("no io error");
+        let list = List::from_file(dir_path, None, false, &mut buf, Dummy).expect("no io error");
         assert!(list.is_none(), "directories are ignored just like Git does it");
 
         Ok(())

gix-ignore/src/lib.rs

@@ -48,6 +48,10 @@ pub enum Kind {
 pub mod parse;
 
 /// Parse git ignore patterns, line by line, from `bytes`.
-pub fn parse(bytes: &[u8]) -> parse::Lines<'_> {
-    parse::Lines::new(bytes)
+///
+/// If `support_precious` is `true`, we will parse `$` prefixed entries as precious.
+/// This is backward-incompatible as files that actually start with `$` like `$houdini`
+/// will then not be ignored anymore, instead it ignores `houdini`.
+pub fn parse(bytes: &[u8], support_precious: bool) -> parse::Lines<'_> {
+    parse::Lines::new(bytes, support_precious)
 }

gix-ignore/src/parse.rs

@@ -4,15 +4,22 @@ use bstr::ByteSlice;
 pub struct Lines<'a> {
     lines: bstr::Lines<'a>,
     line_no: usize,
+    /// Only if `true` we will be able to parse precious files.
+    support_precious: bool,
 }
 
 impl<'a> Lines<'a> {
     /// Create a new instance from `buf` to parse ignore patterns from.
-    pub fn new(buf: &'a [u8]) -> Self {
+    ///
+    /// If `support_precious` is `true`, we will parse `$` prefixed entries as precious.
+    /// This is backward-incompatible as files that actually start with `$` like `$houdini`
+    /// will then not be ignored anymore, instead it ignores `houdini`.
+    pub fn new(buf: &'a [u8], support_precious: bool) -> Self {
         let bom = unicode_bom::Bom::from(buf);
         Lines {
             lines: buf[bom.len()..].lines(),
             line_no: 0,
+            support_precious,
         }
     }
 }
@@ -27,7 +34,7 @@ impl Iterator for Lines<'_> {
                 Some(b'#') | None => continue,
                 Some(c) => c,
             };
-            let (kind, can_negate) = if first == b'$' {
+            let (kind, can_negate) = if self.support_precious && first == b'$' {
                 line = &line[1..];
                 (crate::Kind::Precious, false)
             } else {

gix-ignore/src/search.rs

@@ -21,15 +21,20 @@ pub struct Match<'a> {
     pub sequence_number: usize,
 }
 
-/// An implementation of the [`Pattern`] trait for ignore patterns.
-#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Default)]
-pub struct Ignore;
+/// An implementation of the [`Pattern`] trait for ignore-patterns.
+#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
+pub struct Ignore {
+    /// If `support_precious` is `true`, we will parse `$` prefixed entries as precious.
+    /// This is backward-incompatible as files that actually start with `$` like `$houdini`
+    /// will then not be ignored anymore, instead it ignores `houdini`.
+    pub support_precious: bool,
+}
 
 impl Pattern for Ignore {
     type Value = crate::Kind;
 
-    fn bytes_to_patterns(bytes: &[u8], _source: &std::path::Path) -> Vec<pattern::Mapping<Self::Value>> {
-        crate::parse(bytes)
+    fn bytes_to_patterns(&self, bytes: &[u8], _source: &std::path::Path) -> Vec<pattern::Mapping<Self::Value>> {
+        crate::parse(bytes, self.support_precious)
             .map(|(pattern, line_number, kind)| pattern::Mapping {
                 pattern,
                 value: kind,
@@ -44,38 +49,48 @@ impl Search {
     /// Given `git_dir`, a `.git` repository, load static ignore patterns from `info/exclude`
     /// and from `excludes_file` if it is provided.
     /// Note that it's not considered an error if the provided `excludes_file` does not exist.
-    pub fn from_git_dir(git_dir: &Path, excludes_file: Option<PathBuf>, buf: &mut Vec<u8>) -> std::io::Result<Self> {
+    /// `parse` is a way to parse bytes to ignore patterns.
+    pub fn from_git_dir(
+        git_dir: &Path,
+        excludes_file: Option<PathBuf>,
+        buf: &mut Vec<u8>,
+        parse: Ignore,
+    ) -> std::io::Result<Self> {
         let mut group = Self::default();
 
         let follow_symlinks = true;
         // order matters! More important ones first.
         group.patterns.extend(
             excludes_file
-                .and_then(|file| pattern::List::<Ignore>::from_file(file, None, follow_symlinks, buf).transpose())
+                .and_then(|file| {
+                    pattern::List::<Ignore>::from_file(file, None, follow_symlinks, buf, parse).transpose()
+                })
                 .transpose()?,
         );
         group.patterns.extend(pattern::List::<Ignore>::from_file(
             git_dir.join("info").join("exclude"),
             None,
             follow_symlinks,
             buf,
+            parse,
         )?);
         Ok(group)
     }
 
     /// Parse a list of ignore patterns, using slashes as path separators.
-    pub fn from_overrides(patterns: impl IntoIterator<Item = impl Into<OsString>>) -> Self {
-        Self::from_overrides_inner(&mut patterns.into_iter().map(Into::into))
+    /// `parse` is a way to parse bytes to ignore patterns.
+    pub fn from_overrides(patterns: impl IntoIterator<Item = impl Into<OsString>>, parse: Ignore) -> Self {
+        Self::from_overrides_inner(&mut patterns.into_iter().map(Into::into), parse)
     }
 
-    fn from_overrides_inner(patterns: &mut dyn Iterator<Item = OsString>) -> Self {
+    fn from_overrides_inner(patterns: &mut dyn Iterator<Item = OsString>, parse: Ignore) -> Self {
         Search {
             patterns: vec![pattern::List {
                 patterns: patterns
                     .enumerate()
                     .filter_map(|(seq_id, pattern)| {
                         let pattern = gix_path::try_into_bstr(PathBuf::from(pattern)).ok()?;
-                        crate::parse(pattern.as_ref())
+                        crate::parse(pattern.as_ref(), parse.support_precious)
                             .next()
                             .map(|(p, _seq_id, kind)| pattern::Mapping {
                                 pattern: p,
@@ -95,9 +110,16 @@ impl Search {
 impl Search {
     /// Add patterns as parsed from `bytes`, providing their `source` path and possibly their `root` path, the path they
     /// are relative to. This also means that `source` is contained within `root` if `root` is provided.
-    pub fn add_patterns_buffer(&mut self, bytes: &[u8], source: impl Into<PathBuf>, root: Option<&Path>) {
+    /// Use `parse` to control how ignore patterns are parsed.
+    pub fn add_patterns_buffer(
+        &mut self,
+        bytes: &[u8],
+        source: impl Into<PathBuf>,
+        root: Option<&Path>,
+        parse: Ignore,
+    ) {
         self.patterns
-            .push(pattern::List::from_bytes(bytes, source.into(), root));
+            .push(pattern::List::from_bytes(bytes, source.into(), root, parse));
     }
 }
 

gix-ignore/tests/ignore.rs renamed to gix-ignore/tests/ignore/main.rs

@@ -5,7 +5,7 @@ use gix_testtools::fixture_bytes;
 #[test]
 fn precious() {
     let input = fixture_bytes("ignore/precious.txt");
-    let actual: Vec<_> = gix_ignore::parse(&input).map(flat_map).collect();
+    let actual: Vec<_> = gix_ignore::parse(&input, true).map(flat_map).collect();
     assert_eq!(
         actual,
         vec![
@@ -16,20 +16,32 @@ fn precious() {
             pat_precious("!/*", Mode::empty(), 12),
         ]
     );
+
+    let actual: Vec<_> = gix_ignore::parse(&input, false).map(flat_map).collect();
+    assert_eq!(
+        actual,
+        vec![
+            pat("$.config", Mode::NO_SUB_DIR, 1),
+            pat("$starts-with-dollar", Mode::NO_SUB_DIR, 2),
+            pat("$*.html", Mode::NO_SUB_DIR, 4),
+            pat("foo.html", Mode::NO_SUB_DIR | Mode::NEGATIVE, 6),
+            pat("$!/*", Mode::empty(), 12),
+        ]
+    );
 }
 
 #[test]
 fn byte_order_marks_are_no_patterns() {
     assert_eq!(
-        flatten(gix_ignore::parse("\u{feff}hello".as_bytes()).next()),
+        flatten(gix_ignore::parse("\u{feff}hello".as_bytes(), false).next()),
         Some(pat(r"hello", Mode::NO_SUB_DIR, 1))
     );
 }
 
 #[test]
 fn line_numbers_are_counted_correctly() {
     let input = fixture_bytes("ignore/various.txt");
-    let actual: Vec<_> = gix_ignore::parse(&input).map(flat_map).collect();
+    let actual: Vec<_> = gix_ignore::parse(&input, false).map(flat_map).collect();
     assert_eq!(
         actual,
         vec![
@@ -47,7 +59,7 @@ fn line_numbers_are_counted_correctly() {
 #[test]
 fn line_endings_can_be_windows_or_unix() {
     assert_eq!(
-        gix_ignore::parse(b"unix\nwindows\r\nlast")
+        gix_ignore::parse(b"unix\nwindows\r\nlast", false)
             .map(flat_map)
             .collect::<Vec<_>>(),
         vec![
@@ -60,22 +72,22 @@ fn line_endings_can_be_windows_or_unix() {
 
 #[test]
 fn comments_are_ignored_as_well_as_empty_ones() {
-    assert!(gix_ignore::parse(b"# hello world").next().is_none());
-    assert!(gix_ignore::parse(b"\n\r\n\t\t   \n").next().is_none());
+    assert!(gix_ignore::parse(b"# hello world", false).next().is_none());
+    assert!(gix_ignore::parse(b"\n\r\n\t\t   \n", false).next().is_none());
 }
 
 #[test]
 fn backslashes_before_hashes_are_no_comments() {
     assert_eq!(
-        flatten(gix_ignore::parse(br"\#hello").next()),
+        flatten(gix_ignore::parse(br"\#hello", false).next()),
         Some(pat(r"#hello", Mode::NO_SUB_DIR, 1))
     );
 }
 
 #[test]
 fn trailing_spaces_can_be_escaped_to_be_literal() {
     fn parse_one(input: &str) -> (BString, Mode, usize, gix_ignore::Kind) {
-        let actual: Vec<_> = gix_ignore::parse(input.as_bytes()).map(flat_map).collect();
+        let actual: Vec<_> = gix_ignore::parse(input.as_bytes(), false).map(flat_map).collect();
         assert_eq!(actual.len(), 1, "{input:?} should match");
         actual.into_iter().next().expect("present")
     }