Prototype storing cache in git notes

Change: notes-cache

Author: christian-schilling

josh-core/src/cache.rs

@@ -1,18 +1,66 @@
 use super::*;
 use std::collections::HashMap;
+use std::sync::{LazyLock, RwLock};
 
 const CACHE_VERSION: u64 = 24;
 
+fn josh_commit_signature<'a>() -> JoshResult<git2::Signature<'a>> {
+    Ok(if let Ok(time) = std::env::var("JOSH_COMMIT_TIME") {
+        git2::Signature::new(
+            "JOSH",
+            "josh@josh-project.dev",
+            &git2::Time::new(time.parse()?, 0),
+        )?
+    } else {
+        git2::Signature::now("JOSH", "josh@josh-project.dev")?
+    })
+}
+
+fn store_note(repo: &git2::Repository, kind: &str, key: git2::Oid, from: git2::Oid, to: git2::Oid) {
+    let signature = josh_commit_signature().unwrap();
+    repo.note(
+        &signature,
+        &signature,
+        Some(&format!("refs/josh/{}/{}/{}", kind, CACHE_VERSION, key)),
+        from,
+        &format!("{}", to),
+        true,
+    )
+    .unwrap();
+}
+
+fn lookup_note(
+    repo: &git2::Repository,
+    kind: &str,
+    key: git2::Oid,
+    from: git2::Oid,
+) -> Option<git2::Oid> {
+    if from.as_bytes()[0] != 0 {
+        return None;
+    }
+    if let Ok(note) = repo.find_note(
+        Some(&format!("refs/josh/{}/{}/{}", kind, CACHE_VERSION, key)),
+        from,
+    ) {
+        Some(git2::Oid::from_str(note.message().unwrap()).unwrap())
+    } else {
+        None
+    }
+}
+
 lazy_static! {
     static ref DB: std::sync::Mutex<Option<sled::Db>> = std::sync::Mutex::new(None);
-    static ref REF_CACHE: std::sync::Mutex<HashMap<git2::Oid, HashMap<git2::Oid, git2::Oid>>> =
-        std::sync::Mutex::new(HashMap::new());
-    static ref POPULATE_MAP: std::sync::Mutex<HashMap<(git2::Oid, git2::Oid), git2::Oid>> =
-        std::sync::Mutex::new(HashMap::new());
-    static ref GLOB_MAP: std::sync::Mutex<HashMap<(git2::Oid, git2::Oid), git2::Oid>> =
-        std::sync::Mutex::new(HashMap::new());
 }
 
+static REF_CACHE: LazyLock<RwLock<HashMap<git2::Oid, HashMap<git2::Oid, git2::Oid>>>> =
+    LazyLock::new(Default::default);
+
+static POPULATE_MAP: LazyLock<RwLock<HashMap<(git2::Oid, git2::Oid), git2::Oid>>> =
+    LazyLock::new(Default::default);
+
+static GLOB_MAP: LazyLock<RwLock<HashMap<(git2::Oid, git2::Oid), git2::Oid>>> =
+    LazyLock::new(Default::default);
+
 pub fn load(path: &std::path::Path) -> JoshResult<()> {
     *DB.lock()? = Some(
         sled::Config::default()
@@ -272,32 +320,32 @@ impl Transaction {
     }
 
     pub fn insert_populate(&self, tree: (git2::Oid, git2::Oid), result: git2::Oid) {
-        POPULATE_MAP.lock().unwrap().entry(tree).or_insert(result);
+        POPULATE_MAP.write().unwrap().entry(tree).or_insert(result);
     }
 
     pub fn get_populate(&self, tree: (git2::Oid, git2::Oid)) -> Option<git2::Oid> {
-        return POPULATE_MAP.lock().unwrap().get(&tree).cloned();
+        POPULATE_MAP.read().unwrap().get(&tree).cloned()
     }
 
     pub fn insert_glob(&self, tree: (git2::Oid, git2::Oid), result: git2::Oid) {
-        GLOB_MAP.lock().unwrap().entry(tree).or_insert(result);
+        GLOB_MAP.write().unwrap().entry(tree).or_insert(result);
     }
 
     pub fn get_glob(&self, tree: (git2::Oid, git2::Oid)) -> Option<git2::Oid> {
-        return GLOB_MAP.lock().unwrap().get(&tree).cloned();
+        GLOB_MAP.read().unwrap().get(&tree).cloned()
     }
 
     pub fn insert_ref(&self, filter: filter::Filter, from: git2::Oid, to: git2::Oid) {
         REF_CACHE
-            .lock()
+            .write()
             .unwrap()
             .entry(filter.id())
             .or_default()
             .insert(from, to);
     }
 
     pub fn get_ref(&self, filter: filter::Filter, from: git2::Oid) -> Option<git2::Oid> {
-        if let Some(m) = REF_CACHE.lock().unwrap().get(&filter.id()) {
+        if let Some(m) = REF_CACHE.read().unwrap().get(&filter.id()) {
             if let Some(oid) = m.get(&from) {
                 if self.repo.odb().unwrap().exists(*oid) {
                     return Some(*oid);
@@ -336,6 +384,9 @@ impl Transaction {
             });
 
             t.insert(from.as_bytes(), to.as_bytes()).unwrap();
+            if from.as_bytes()[0] == 0 {
+                store_note(&self.repo, "cache", filter.id(), from, to);
+            }
         }
     }
 
@@ -396,8 +447,15 @@ impl Transaction {
                 .open_tree(filter::spec(filter))
                 .unwrap()
         });
-        if let Some(oid) = t.get(from.as_bytes()).unwrap() {
-            let oid = git2::Oid::from_bytes(&oid).unwrap();
+        let oid = if let Some(oid) = t.get(from.as_bytes()).unwrap() {
+            Some(git2::Oid::from_bytes(&oid).unwrap())
+        } else if let Some(oid) = lookup_note(&self.repo, "cache", filter.id(), from) {
+            Some(oid)
+        } else {
+            None
+        };
+
+        if let Some(oid) = oid {
             if oid == git2::Oid::zero() {
                 return Some(oid);
             }

tests/proxy/workspace_modify_chain.t

@@ -358,6 +358,7 @@
       |   |-- 20
       |   |   `-- 5dccdc2e37df9aaccec46009520154219134e7
       |   |-- 27
+      |   |   |-- 153f2bc72430a7221a94318d46d2eb1402c9db
       |   |   `-- d0eee16bdbe4a1ade0ebf877f97467de3b218e
       |   |-- 2a
       |   |   |-- 03ad0fe1720ee0afc95ba8e1bc38a35b87983f
@@ -383,6 +384,8 @@
       |   |   `-- 3b7672b3bcf00bbd95e650076c375528d5f5c3
       |   |-- 47
       |   |   `-- 8644b35118f1d733b14cafb04c51e5b6579243
+      |   |-- 4a
+      |   |   `-- 63d3ce0a2d493bd0cda58ae8fd064be63c9e95
       |   |-- 4b
       |   |   `-- 825dc642cb6eb9a060e54bf8d69288fbee4904
       |   |-- 4f
@@ -431,7 +434,8 @@
       |   |-- a3
       |   |   `-- d027b4cc082c08af95fcbe03eecd9a62a08c48
       |   |-- a7
-      |   |   `-- 3af649c49fa4b0facff36fafdc1e2bef594d4e
+      |   |   |-- 3af649c49fa4b0facff36fafdc1e2bef594d4e
+      |   |   `-- fe5239814b5bfdbadc9f213abc2a2f2c1f31db
       |   |-- aa
       |   |   `-- 0654329f0642e4505db87c24aea80ba52fd689
       |   |-- ac
@@ -484,9 +488,13 @@
       |   `-- pack
       `-- refs
           |-- heads
+          |-- josh
+          |   `-- cache
+          |       `-- 22
+          |           `-- 7bd92d97e96693ea7fd7eb5757b3580002889948
           |-- namespaces
           `-- tags
 
-  122 directories, 117 files
+  126 directories, 121 files
 
 $ cat ${TESTTMP}/josh-proxy.out | grep VIEW