Implement symref (HEAD) forwarding (#543)

* Implement symref (HEAD) forwarding

Previously it was simply assumed that HEAD always points to
refs/heads/master but as it turns out that is far from true.
Now HEAD is queried per repo from the upstream and forwarded.
Unfortunately it seems that it is not avoidable to to an extra
request to the upstream server for this. (calling ls-remote)

* Silence output of merges in tests

With the new git release sometimes the merge will be done by
"ort" instead of "recursive". We don't really care about that
as the results are the same.

* Remove expect for broken refs

Author: christian-schilling

josh-proxy/src/bin/josh-proxy.rs

@@ -34,12 +34,15 @@ type FetchTimers = HashMap<String, std::time::Instant>;
 type Polls =
     Arc<std::sync::Mutex<std::collections::HashSet<(String, josh_proxy::auth::Handle, String)>>>;
 
+type HeadsMap = Arc<std::sync::RwLock<std::collections::HashMap<String, String>>>;
+
 #[derive(Clone)]
 struct JoshProxyService {
     port: String,
     repo_path: std::path::PathBuf,
     upstream_url: String,
     fetch_timers: Arc<RwLock<FetchTimers>>,
+    heads_map: HeadsMap,
     fetch_permits: Arc<tokio::sync::Semaphore>,
     filter_permits: Arc<tokio::sync::Semaphore>,
     poll: Polls,
@@ -66,10 +69,10 @@ async fn fetch_upstream(
     let auth = auth.clone();
     let key = remote_url.clone();
 
-    let refs_to_fetch = if !headref.is_empty() && !headref.starts_with("refs/heads/") {
-        vec!["refs/heads/*", "refs/tags/*", headref]
+    let refs_to_fetch = if headref != "HEAD" && !headref.starts_with("refs/heads/") {
+        vec!["HEAD*", "refs/heads/*", "refs/tags/*", headref]
     } else {
-        vec!["refs/heads/*", "refs/tags/*"]
+        vec!["HEAD*", "refs/heads/*", "refs/tags/*"]
     };
 
     let refs_to_fetch: Vec<_> = refs_to_fetch.iter().map(|x| x.to_string()).collect();
@@ -113,6 +116,7 @@ async fn fetch_upstream(
     }
 
     let fetch_timers = service.fetch_timers.clone();
+    let heads_map = service.heads_map.clone();
     let br_path = service.repo_path.clone();
 
     let s = tracing::span!(tracing::Level::TRACE, "fetch worker");
@@ -126,6 +130,21 @@ async fn fetch_upstream(
     })
     .await?;
 
+    let us = upstream_repo.clone();
+    let s = tracing::span!(tracing::Level::TRACE, "get_head worker");
+    let br_path = service.repo_path.clone();
+    let ru = remote_url.clone();
+    let a = auth.clone();
+    let hres = tokio::task::spawn_blocking(move || {
+        let _e = s.enter();
+        josh_proxy::get_head(&br_path, &ru, &a)
+    })
+    .await?;
+
+    if let Ok(hres) = hres {
+        heads_map.write()?.insert(us, hres);
+    }
+
     std::mem::drop(permit);
 
     if let Ok(res) = res {
@@ -227,6 +246,7 @@ async fn do_filter(
     headref: String,
 ) -> josh::JoshResult<()> {
     let permit = service.filter_permits.acquire().await;
+    let heads_map = service.heads_map.clone();
 
     let s = tracing::span!(tracing::Level::TRACE, "do_filter worker");
     let r = tokio::task::spawn_blocking(move || {
@@ -273,13 +293,25 @@ async fn do_filter(
             temp_ns.reference(&headref),
         ));
 
+        let mut headref = headref;
+
         josh::filter_refs(&transaction, filter, &from_to)?;
-        transaction.repo().reference_symbolic(
-            &temp_ns.reference("HEAD"),
-            &temp_ns.reference(&headref),
-            true,
-            "",
-        )?;
+        if headref == "HEAD" {
+            headref = heads_map
+                .read()?
+                .get(&upstream_repo)
+                .unwrap_or(&"invalid".to_string())
+                .clone();
+        }
+        transaction
+            .repo()
+            .reference_symbolic(
+                &temp_ns.reference("HEAD"),
+                &temp_ns.reference(&headref),
+                true,
+                "",
+            )
+            .ok();
         Ok(())
     })
     .await?;
@@ -368,17 +400,14 @@ async fn call_service(
         } else {
             return Ok(Response::builder()
                 .status(302)
-                .header(
-                    "Location",
-                    format!("/~/browse{}@refs/heads/master(:/)/()", path),
-                )
+                .header("Location", format!("/~/browse{}@HEAD(:/)/()", path))
                 .body(hyper::Body::empty())?);
         }
     };
 
     let mut headref = parsed_url.headref.trim_start_matches('@').to_owned();
     if headref.is_empty() {
-        headref = "refs/heads/master".to_string();
+        headref = "HEAD".to_string();
     }
 
     let remote_url = [
@@ -611,6 +640,7 @@ async fn run_proxy() -> josh::JoshResult<i32> {
         repo_path: local.to_owned(),
         upstream_url: remote.to_owned(),
         fetch_timers: Arc::new(RwLock::new(FetchTimers::new())),
+        heads_map: Arc::new(RwLock::new(std::collections::HashMap::new())),
         poll: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
         fetch_permits: Arc::new(tokio::sync::Semaphore::new(
             ARGS.value_of("n").unwrap_or("1").parse()?,

josh-proxy/src/lib.rs

@@ -340,6 +340,34 @@ fn url_with_auth(url: &str, username: &str) -> String {
     }
 }
 
+pub fn get_head(
+    path: &std::path::Path,
+    url: &str,
+    auth: &auth::Handle,
+) -> josh::JoshResult<String> {
+    let shell = josh::shell::Shell {
+        cwd: path.to_owned(),
+    };
+    let (username, password) = auth.parse()?;
+    let nurl = url_with_auth(url, &username);
+
+    let cmd = format!("git ls-remote --symref {} {}", &nurl, "HEAD");
+    tracing::info!("get_head {:?} {:?} {:?}", cmd, path, "");
+
+    let (stdout, _stderr, _) = shell.command_env(&cmd, &[], &[("GIT_PASSWORD", &password)]);
+
+    let head = stdout
+        .lines()
+        .next()
+        .unwrap_or("refs/heads/master")
+        .to_string();
+
+    let head = head.replacen("ref: ", "", 1);
+    let head = head.replacen("\tHEAD", "", 1);
+
+    Ok(head)
+}
+
 pub fn fetch_refs_from_url(
     path: &std::path::Path,
     upstream_repo: &str,

src/housekeeping.rs

@@ -30,7 +30,7 @@ pub fn default_from_to(
     }
     refs.append(&mut memorize_from_to(
         repo,
-        &crate::to_filtered_ref(upstream_repo, filter_spec),
+        &to_filtered_ref(upstream_repo, filter_spec),
         upstream_repo,
     ));
 
@@ -43,13 +43,10 @@ pub fn memorize_from_to(
     upstream_repo: &str,
 ) -> Vec<(String, String)> {
     let mut refs = vec![];
-    let glob = format!(
-        "refs/josh/upstream/{}/refs/heads/master",
-        &to_ns(upstream_repo)
-    );
+    let glob = format!("refs/josh/upstream/{}/HEAD", &to_ns(upstream_repo));
     for refname in repo.references_glob(&glob).unwrap().names() {
         let refname = refname.unwrap();
-        let to_ref = format!("refs/{}/heads/master", &namespace);
+        let to_ref = format!("refs/{}/HEAD", &namespace);
 
         refs.push((refname.to_owned(), to_ref.clone()));
     }

tests/filter/ambigous_merge.t

@@ -72,13 +72,7 @@
   * 81a8353 add sub_/fileY
   * 7671c2a add sub1/file1
 
-  $ git merge hidden_branch1 --no-ff
-  Merge made by the 'recursive' strategy.
-   sub1/file3 | 1 +
-   sub1/fileY | 1 +
-   2 files changed, 2 insertions(+)
-   create mode 100644 sub1/file3
-   create mode 100644 sub1/fileY
+  $ git merge -q hidden_branch1 --no-ff
   $ git log --graph --oneline
   *   2fcb6a4 Merge branch 'hidden_branch1' into hidden_master
   |\  

tests/filter/empty_orphan.t

@@ -56,13 +56,7 @@ Empty root commits from unrelated parts of the tree should not be included
 
   $ git checkout master
   Switched to branch 'master'
-  $ git merge other --no-ff --allow-unrelated
-  Merge made by the 'recursive' strategy.
-   some_file       | 1 +
-   some_other_file | 1 +
-   2 files changed, 2 insertions(+)
-   create mode 100644 some_file
-   create mode 100644 some_other_file
+  $ git merge -q other --no-ff --allow-unrelated
 
   $ tree
   .

tests/filter/empty_reimport.t

@@ -26,13 +26,7 @@
   $ git add .
   $ git commit -m "unrelated change on this" 1> /dev/null
 
-  $ git merge other_branch
-  Merge made by the 'recursive' strategy.
-   pre/testfile2 | 1 +
-   pre/testfile4 | 1 +
-   2 files changed, 2 insertions(+)
-   create mode 100644 pre/testfile2
-   create mode 100644 pre/testfile4
+  $ git merge -q other_branch
   $ git log --graph --pretty=%s
   *   Merge branch 'other_branch'
   |\  
@@ -68,13 +62,7 @@
   $ echo content > testfile7
   $ git add .
   $ git commit -m "more unrelated change on this" 1> /dev/null
-  $ git merge other_branch
-  Merge made by the 'recursive' strategy.
-   pre/blafile1 | 1 +
-   pre/blefile2 | 1 +
-   2 files changed, 2 insertions(+)
-   create mode 100644 pre/blafile1
-   create mode 100644 pre/blefile2
+  $ git merge -q other_branch
 
   $ git log --graph --pretty=%s
   *   Merge branch 'other_branch'

tests/filter/reverse_merge.t

@@ -65,11 +65,7 @@
   * add file3
   * add file1
 
-  $ git merge hidden_branch1 --no-ff
-  Merge made by the 'recursive' strategy.
-   sub1/file3 | 1 +
-   1 file changed, 1 insertion(+)
-   create mode 100644 sub1/file3
+  $ git merge -q hidden_branch1 --no-ff
   $ git log --graph --pretty=%s
   *   Merge branch 'hidden_branch1' into hidden_master
   |\  

tests/proxy/amend_patchset.t

@@ -125,13 +125,12 @@
   |   |-- filtered
   |   |   `-- real_repo.git
   |   |       |-- %3A
-  |   |       |   `-- heads
-  |   |       |       `-- master
+  |   |       |   `-- HEAD
   |   |       `-- %3A%2Fsub3
-  |   |           `-- heads
-  |   |               `-- master
+  |   |           `-- HEAD
   |   `-- upstream
   |       `-- real_repo.git
+  |           |-- HEAD
   |           `-- refs
   |               |-- changes
   |               |   `-- 1
@@ -143,4 +142,4 @@
   |-- namespaces
   `-- tags
 
-  17 directories, 5 files
+  15 directories, 6 files

tests/proxy/authentication.t

@@ -128,17 +128,16 @@
   |   |-- filtered
   |   |   `-- real_repo.git
   |   |       |-- %3A
-  |   |       |   `-- heads
-  |   |       |       `-- master
+  |   |       |   `-- HEAD
   |   |       `-- %3A%2Fsub1
-  |   |           `-- heads
-  |   |               `-- master
+  |   |           `-- HEAD
   |   `-- upstream
   |       `-- real_repo.git
+  |           |-- HEAD
   |           `-- refs
   |               `-- heads
   |                   `-- master
   |-- namespaces
   `-- tags
 
-  14 directories, 3 files
+  12 directories, 4 files

tests/proxy/clone_absent_head.t

@@ -88,13 +88,9 @@
   refs
   |-- heads
   |-- josh
-  |   `-- filtered
-  |       `-- real_repo.git
-  |           `-- %3A
-  |               `-- heads
-  |                   `-- master
   |-- namespaces
   `-- tags
 
-  8 directories, 1 file
+  4 directories, 0 files
 
+$ cat ${TESTTMP}/josh-proxy.out