Implement :from filter

Change: from-filter

Author: christian-schilling

docs/src/reference/filters.md

@@ -114,13 +114,34 @@ commits that don't match any of the other shas.
 Produce the history that would be the result of pushing the passed branches with the
 passed filters into the upstream.
 
+### Start filtering from a specific commit **:from(<sha>:filter)**
+
+Produce a history that keeps the original history leading up to the specified commit `<sha>` unchanged,
+but applies the given `:filter` to all commits from that commit onwards.
+
 ### Prune trivial merge commits **:prune=trivial-merge**
 
 Produce a history that skips all merge commits whose tree is identical to the first parents
 tree.
 Normally Josh will keep all commits in the filtered history whose tree differs from any of it's
 parents.
 
+### Pin tree contents
+
+`:pin` filter prevents appearance of selected subtrees for a given revision.
+
+In practical terms, it means that file and folder updates are "held off", and revisions are "pinned".
+If a tree entry already existed in the parent revision, that version will be chosen.
+Otherwise, the tree entry will not appear in the filtered commit.
+
+The source of the parent revision is always the first commit parent.
+
+Note that this filter is only practical when used with `:hook` or `workspace.josh`,
+as it should apply per-revision only. Applying `:pin` for the whole history
+will result in the subtree being excluded from all revisions.
+
+Refer to `pin_filter_workspace.t` and `pin_filter_hook.t` for reference.
+
 Filter order matters
 --------------------
 

josh-core/src/filter/grammar.pest

@@ -24,6 +24,8 @@ filter_spec = { (
     filter_group
   | filter_message
   | filter_rev
+  | filter_from
+  | filter_concat
   | filter_join
   | filter_replace
   | filter_squash
@@ -51,6 +53,24 @@ filter_rev = {
     ~ ")"
 }
 
+filter_from = {
+    CMD_START ~ "from" ~ "("
+    ~ NEWLINE*
+    ~ (rev ~ filter_spec)?
+    ~ (CMD_SEP+ ~ (rev ~ filter_spec))*
+    ~ NEWLINE*
+    ~ ")"
+}
+
+filter_concat = {
+    CMD_START ~ "from" ~ "("
+    ~ NEWLINE*
+    ~ (rev ~ filter_spec)?
+    ~ (CMD_SEP+ ~ (rev ~ filter_spec))*
+    ~ NEWLINE*
+    ~ ")"
+}
+
 filter_join = {
     CMD_START ~ "join" ~ "("
     ~ NEWLINE*
@@ -60,7 +80,6 @@ filter_join = {
     ~ ")"
 }
 
-
 filter_replace = {
     CMD_START ~ "replace" ~ "("
     ~ NEWLINE*

josh-core/src/filter/mod.rs

@@ -300,10 +300,13 @@ enum Op {
     Pattern(String),
     Message(String),
 
+    HistoryConcat(LazyRef, Filter),
+
     Compose(Vec<Filter>),
     Chain(Filter, Filter),
     Subtract(Filter, Filter),
     Exclude(Filter),
+    Pin(Filter),
 }
 
 /// Pretty print the filter on multiple lines with initial indentation level.
@@ -349,6 +352,10 @@ fn pretty2(op: &Op, indent: usize, compose: bool) -> String {
             Op::Compose(filters) => ff(&filters, "exclude", indent),
             b => format!(":exclude[{}]", pretty2(&b, indent, false)),
         },
+        Op::Pin(filter) => match to_op(*filter) {
+            Op::Compose(filters) => ff(&filters, "pin", indent),
+            b => format!(":pin[{}]", pretty2(&b, indent, false)),
+        },
         Op::Chain(a, b) => match (to_op(*a), to_op(*b)) {
             (Op::Subdir(p1), Op::Prefix(p2)) if p1 == p2 => {
                 format!("::{}/", parse::quote_if(&p1.to_string_lossy()))
@@ -407,7 +414,7 @@ fn lazy_refs2(op: &Op) -> Vec<String> {
                     acc
                 })
         }
-        Op::Exclude(filter) => lazy_refs(*filter),
+        Op::Exclude(filter) | Op::Pin(filter) => lazy_refs(*filter),
         Op::Chain(a, b) => {
             let mut av = lazy_refs(*a);
             av.append(&mut lazy_refs(*b));
@@ -419,6 +426,13 @@ fn lazy_refs2(op: &Op) -> Vec<String> {
             av
         }
         Op::Rev(filters) => lazy_refs2(&Op::Join(filters.clone())),
+        Op::HistoryConcat(r, _) => {
+            let mut lr = Vec::new();
+            if let LazyRef::Lazy(s) = r {
+                lr.push(s.to_owned());
+            }
+            lr
+        }
         Op::Join(filters) => {
             let mut lr = lazy_refs2(&Op::Compose(filters.values().copied().collect()));
             lr.extend(filters.keys().filter_map(|x| {
@@ -458,6 +472,7 @@ fn resolve_refs2(refs: &std::collections::HashMap<String, git2::Oid>, op: &Op) -
             Op::Compose(filters.iter().map(|f| resolve_refs(refs, *f)).collect())
         }
         Op::Exclude(filter) => Op::Exclude(resolve_refs(refs, *filter)),
+        Op::Pin(filter) => Op::Pin(resolve_refs(refs, *filter)),
         Op::Chain(a, b) => Op::Chain(resolve_refs(refs, *a), resolve_refs(refs, *b)),
         Op::Subtract(a, b) => Op::Subtract(resolve_refs(refs, *a), resolve_refs(refs, *b)),
         Op::Rev(filters) => {
@@ -478,6 +493,19 @@ fn resolve_refs2(refs: &std::collections::HashMap<String, git2::Oid>, op: &Op) -
                 .collect();
             Op::Rev(lr)
         }
+        Op::HistoryConcat(r, filter) => {
+            let f = resolve_refs(refs, *filter);
+            let resolved_ref = if let LazyRef::Lazy(s) = r {
+                if let Some(res) = refs.get(s) {
+                    LazyRef::Resolved(*res)
+                } else {
+                    r.clone()
+                }
+            } else {
+                r.clone()
+            };
+            Op::HistoryConcat(resolved_ref, f)
+        }
         Op::Join(filters) => {
             let lr = filters
                 .iter()
@@ -545,6 +573,9 @@ fn spec2(op: &Op) -> String {
         Op::Exclude(b) => {
             format!(":exclude[{}]", spec(*b))
         }
+        Op::Pin(filter) => {
+            format!(":pin[{}]", spec(*filter))
+        }
         Op::Rev(filters) => {
             let mut v = filters
                 .iter()
@@ -616,6 +647,9 @@ fn spec2(op: &Op) -> String {
         Op::Message(m) => {
             format!(":{}", parse::quote(m))
         }
+        Op::HistoryConcat(r, filter) => {
+            format!(":concat({}{})", r.to_string(), spec(*filter))
+        }
         Op::Hook(hook) => {
             format!(":hook={}", parse::quote(hook))
         }
@@ -688,6 +722,9 @@ fn as_tree2(repo: &git2::Repository, op: &Op) -> JoshResult<git2::Oid> {
         Op::Exclude(b) => {
             builder.insert("exclude", as_tree(repo, *b)?, git2::FileMode::Tree.into())?;
         }
+        Op::Pin(b) => {
+            builder.insert("pin", as_tree(repo, *b)?, git2::FileMode::Tree.into())?;
+        }
         Op::Subdir(path) => {
             builder.insert(
                 "subdir",
@@ -808,6 +845,13 @@ fn as_tree2(repo: &git2::Repository, op: &Op) -> JoshResult<git2::Oid> {
             v.sort();
             builder.insert("rev", rev_params(repo, &v)?, git2::FileMode::Tree.into())?;
         }
+        Op::HistoryConcat(r, f) => {
+            builder.insert(
+                "historyconcat",
+                rev_params(repo, &vec![(r.to_string(), *f)])?,
+                git2::FileMode::Tree.into(),
+            )?;
+        }
         Op::Join(filters) => {
             let mut v = filters
                 .iter()
@@ -1064,6 +1108,11 @@ fn from_tree2(repo: &git2::Repository, tree_oid: git2::Oid) -> JoshResult<Op> {
             let filter = from_tree2(repo, exclude_tree.id())?;
             Ok(Op::Exclude(to_filter(filter)))
         }
+        "pin" => {
+            let pin_tree = repo.find_tree(entry.id())?;
+            let filter = from_tree2(repo, pin_tree.id())?;
+            Ok(Op::Pin(to_filter(filter)))
+        }
         "rev" => {
             let rev_tree = repo.find_tree(entry.id())?;
             let mut filters = std::collections::BTreeMap::new();
@@ -1608,6 +1657,19 @@ fn apply_to_commit2(
                 parent_filters,
             );
         }
+        Op::HistoryConcat(r, f) => {
+            if let LazyRef::Resolved(c) = r {
+                let a = apply_to_commit2(&to_op(*f), &repo.find_commit(*c)?, transaction)?;
+                let a = some_or!(a, { return Ok(None) });
+                if commit.id() == a {
+                    transaction.insert(filter, commit.id(), *c, true);
+                    return Ok(Some(*c));
+                }
+            } else {
+                return Err(josh_error("unresolved lazy ref"));
+            }
+            Apply::from_commit(commit)?
+        }
         _ => {
             let filtered_parent_ids = commit
                 .parent_ids()
@@ -1659,7 +1721,7 @@ fn apply2<'a>(transaction: &'a cache::Transaction, op: &Op, x: Apply<'a>) -> Jos
         Op::Nop => Ok(x),
         Op::Empty => Ok(x.with_tree(tree::empty(repo))),
         Op::Fold => Ok(x),
-        Op::Squash(None) => Ok(x),
+        Op::Squash(..) => Ok(x),
         Op::Author(author, email) => Ok(x.with_author((author.clone(), email.clone()))),
         Op::Committer(author, email) => Ok(x.with_committer((author.clone(), email.clone()))),
         Op::Message(m) => Ok(x.with_message(
@@ -1669,7 +1731,7 @@ fn apply2<'a>(transaction: &'a cache::Transaction, op: &Op, x: Apply<'a>) -> Jos
                 &std::collections::HashMap::<String, &dyn strfmt::DisplayStr>::new(),
             )?,
         )),
-        Op::Squash(Some(_)) => Err(josh_error("not applicable to tree")),
+        Op::HistoryConcat(..) => Ok(x),
         Op::Linear => Ok(x),
         Op::Prune => Ok(x),
         Op::Unsign => Ok(x),
@@ -1784,6 +1846,25 @@ fn apply2<'a>(transaction: &'a cache::Transaction, op: &Op, x: Apply<'a>) -> Jos
             return apply(transaction, *b, apply(transaction, *a, x.clone())?);
         }
         Op::Hook(_) => Err(josh_error("not applicable to tree")),
+
+        Op::Pin(pin_filter) => {
+            let filtered_parent = if let Some(parent) = x.parents.as_ref().and_then(|p| p.first()) {
+                let parent = repo.find_commit(*parent)?;
+                let filtered = apply(transaction, *pin_filter, Apply::from_commit(&parent)?)?;
+                filtered.tree.id()
+            } else {
+                tree::empty_id()
+            };
+
+            // Mask out all the "pinned" files from current tree
+            let exclude = to_filter(Op::Exclude(*pin_filter));
+            let with_mask = apply(transaction, exclude, x.clone())?;
+
+            // Overlay filtered parent tree on current one to override versions
+            let with_overlay = tree::overlay(transaction, with_mask.tree.id(), filtered_parent)?;
+
+            Ok(x.with_tree(repo.find_tree(with_overlay)?))
+        }
     }
 }
 

josh-core/src/filter/opt.rs

@@ -82,6 +82,7 @@ pub fn simplify(filter: Filter) -> Filter {
             Op::Subtract(simplify(to_filter(a)), simplify(to_filter(b)))
         }
         Op::Exclude(b) => Op::Exclude(simplify(b)),
+        Op::Pin(b) => Op::Pin(simplify(b)),
         _ => to_op(filter),
     });
 
@@ -137,6 +138,7 @@ pub fn flatten(filter: Filter) -> Filter {
             Op::Subtract(flatten(to_filter(a)), flatten(to_filter(b)))
         }
         Op::Exclude(b) => Op::Exclude(flatten(b)),
+        Op::Pin(b) => Op::Pin(flatten(b)),
         _ => to_op(filter),
     });
 
@@ -440,8 +442,9 @@ fn step(filter: Filter) -> Filter {
             (a, b) => Op::Chain(step(to_filter(a)), step(to_filter(b))),
         },
         Op::Exclude(b) if b == to_filter(Op::Nop) => Op::Empty,
-        Op::Exclude(b) if b == to_filter(Op::Empty) => Op::Nop,
+        Op::Exclude(b) | Op::Pin(b) if b == to_filter(Op::Empty) => Op::Nop,
         Op::Exclude(b) => Op::Exclude(step(b)),
+        Op::Pin(b) => Op::Pin(step(b)),
         Op::Subtract(a, b) if a == b => Op::Empty,
         Op::Subtract(af, bf) => match (to_op(af), to_op(bf)) {
             (Op::Empty, _) => Op::Empty,
@@ -504,6 +507,7 @@ pub fn invert(filter: Filter) -> JoshResult<Filter> {
         Op::Pattern(pattern) => Some(Op::Pattern(pattern)),
         Op::Rev(_) => Some(Op::Nop),
         Op::RegexReplace(_) => Some(Op::Nop),
+        Op::Pin(_) => Some(Op::Nop),
         _ => None,
     };
 

josh-core/src/filter/parse.rs

@@ -115,6 +115,7 @@ fn parse_item(pair: pest::iterators::Pair<Rule>) -> JoshResult<Op> {
                 [cmd, args] => {
                     let g = parse_group(args)?;
                     match *cmd {
+                        "pin" => Ok(Op::Pin(to_filter(Op::Compose(g)))),
                         "exclude" => Ok(Op::Exclude(to_filter(Op::Compose(g)))),
                         "subtract" if g.len() == 2 => Ok(Op::Subtract(g[0], g[1])),
                         _ => Err(josh_error(&format!("parse_item: no match {:?}", cmd))),
@@ -145,6 +146,31 @@ fn parse_item(pair: pest::iterators::Pair<Rule>) -> JoshResult<Op> {
 
             Ok(Op::Rev(hm))
         }
+        Rule::filter_from => {
+            let v: Vec<_> = pair.into_inner().map(|x| x.as_str()).collect();
+
+            if v.len() == 2 {
+                let oid = LazyRef::parse(v[0])?;
+                let filter = parse(v[1])?;
+                Ok(Op::Chain(
+                    filter,
+                    filter::to_filter(Op::HistoryConcat(oid, filter)),
+                ))
+            } else {
+                Err(josh_error("wrong argument count for :from"))
+            }
+        }
+        Rule::filter_concat => {
+            let v: Vec<_> = pair.into_inner().map(|x| x.as_str()).collect();
+
+            if v.len() == 2 {
+                let oid = LazyRef::parse(v[0])?;
+                let filter = parse(v[1])?;
+                Ok(Op::HistoryConcat(oid, filter))
+            } else {
+                Err(josh_error("wrong argument count for :concat"))
+            }
+        }
         Rule::filter_replace => {
             let replacements = pair
                 .into_inner()

tests/filter/concat.t

@@ -0,0 +1,42 @@
+  $ export TESTTMP=${PWD}
+
+  $ cd ${TESTTMP}
+  $ git init -q libs 1> /dev/null
+  $ cd libs
+
+  $ mkdir sub1
+  $ echo contents1 > sub1/file1
+  $ git add sub1
+  $ git commit -m "add file1" 1> /dev/null
+
+  $ echo contents2 > sub1/file2
+  $ git add sub1
+  $ git commit -m "add file2" 1> /dev/null
+  $ git update-ref refs/heads/from_here HEAD
+
+
+  $ mkdir sub2
+  $ echo contents1 > sub2/file3
+  $ git add sub2
+  $ git commit -m "add file3" 1> /dev/null
+
+  $ josh-filter ":\"x\""
+
+  $ git log --graph --pretty=%s:%H HEAD
+  * add file3:667a912db7482f3c8023082c9b4c7b267792633a
+  * add file2:81b10fb4984d20142cd275b89c91c346e536876a
+  * add file1:bb282e9cdc1b972fffd08fd21eead43bc0c83cb8
+
+  $ git log --graph --pretty=%s:%H FILTERED_HEAD
+  * x:9d117d96dfdba145df43ebe37d9e526acac4b17c
+  * x:b232aa8eefaadfb5e38b3ad7355118aa59fb651e
+  * x:6b4d1f87c2be08f7d0f9d40b6679aab612e259b1
+
+  $ josh-filter -p ":from(81b10fb4984d20142cd275b89c91c346e536876a:\"x\")"
+  :"x":concat(81b10fb4984d20142cd275b89c91c346e536876a:"x")
+  $ josh-filter ":from(81b10fb4984d20142cd275b89c91c346e536876a:\"x\")"
+
+  $ git log --graph --pretty=%s FILTERED_HEAD
+  * x
+  * add file2
+  * add file1