git: Exit clearly when in a "dubious" directory

I hit a _very_ hard to diagnose issue today after switching part of a
CI environment.  It turns out that due to non-privileged container
magic the CI step was not running as the user that cloned the repo.
This manifested as the sdist generated during this step just not
having a few extra data files; but the build still passed.  So several
steps later, suddenly files have just vanished from the sdist, which
of course is not something that is checked for and so testing blows up
in very strange ways.

After I understood what was going on, there is a _tiny_ little message
hidden amongst the logs that gives a hint of what's going on.

...
writing requirements to src/proj.egg-info/requires.txt
writing top-level names to src/proj.egg-info/top_level.txt
listing git files failed - pretending there aren't any
reading manifest file 'src/proj.egg-info/SOURCES.txt'
writing manifest file 'src/proj.egg-info/SOURCES.txt'
...

What is actually happening is that if you run git you get "git status
fatal: detected dubious ownership in repository at '/..proj'".  This
is the well-known CVE-2022-24765 issue where trusting a `.git` config
dir from another user causes problems.  In
6a3bb96 all the calls in
setuptools_scm/git.py were updated to use `--git-dir` directly -- git
will not complain if you have told it to explicitly trust the config
dir like this.

However, there are calls in _file_finders/git.py to find the top-level
that are not using this.  It silently (modulo an easily missed log)
skips adding files when this occurs.

I can not see that this would ever be the behaviour you would want.
If it had of exploded telling me the git call failed, it would have
short-cut all of the problem finding.

This adds an explicit match on the static part of this git failure
message and raises a SystemExit if it hits.  A test case that mocks
such a situation is added.

I have increased the "listing git files failed" to a warning; similar
to the "shallow clone" warning.  It seems like this is actually quite
important.  I'm sure someone relies on it for some reason, but I can't
quite see why you'd want to ignore that files are not being added to
your sdist.  This necessitates ignoring this warning in several
places, which I think is an improvement, as at least we have a clearer
overview of where addition is being ignored if we want to examine this
behaviour more closely.

Closes: #784
Signed-off-by: Ian Wienand <iwienand@redhat.com>

Author: ianw

src/setuptools_scm/_file_finders/git.py

@@ -4,6 +4,7 @@
 import os
 import subprocess
 import tarfile
+import warnings
 
 from typing import IO
 
@@ -22,8 +23,18 @@ def _git_toplevel(path: str) -> str | None:
         cwd = os.path.abspath(path or ".")
         res = _run(["git", "rev-parse", "HEAD"], cwd=cwd)
         if res.returncode:
+            # This catches you being in a git directory, but the
+            # permissions being incorrect.  With modern contanizered
+            # CI environments you can easily end up in a cloned repo
+            # with incorrect permissions and we don't want to silently
+            # ignore files.
+            if "--add safe.directory" in res.stderr:
+                log.error(res.stderr)
+                raise SystemExit(
+                    "git introspection failed: {}".format(res.stderr.split("\n")[0])
+                )
             # BAIL if there is no commit
-            log.error("listing git files failed - pretending there aren't any")
+            warnings.warn("listing git files failed - pretending there aren't any")
             return None
         res = _run(
             ["git", "rev-parse", "--show-prefix"],

testing/test_file_finder.py

@@ -50,18 +50,21 @@ def _sep(paths: Iterable[str]) -> set[str]:
     return {path.replace("/", os.path.sep) for path in paths}
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 def test_basic(inwd: WorkDir) -> None:
     assert set(find_files()) == _sep({"file1", "adir/filea", "bdir/fileb"})
     assert set(find_files(".")) == _sep({"./file1", "./adir/filea", "./bdir/fileb"})
     assert set(find_files("adir")) == _sep({"adir/filea"})
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 def test_whitespace(inwd: WorkDir) -> None:
     (inwd.cwd / "adir" / "space file").touch()
     inwd.add_and_commit()
     assert set(find_files("adir")) == _sep({"adir/space file", "adir/filea"})
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 def test_case(inwd: WorkDir) -> None:
     (inwd.cwd / "CamelFile").touch()
     (inwd.cwd / "file2").touch()
@@ -71,6 +74,7 @@ def test_case(inwd: WorkDir) -> None:
     )
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(
     os.path.normcase("B") != os.path.normcase("b"), reason="case sensitive filesystem"
 )
@@ -84,19 +88,22 @@ def test_case_cwd_evil(inwd: WorkDir, monkeypatch: pytest.MonkeyPatch) -> None:
     )
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(sys.platform == "win32", reason="symlinks to dir not supported")
 def test_symlink_dir(inwd: WorkDir) -> None:
     (inwd.cwd / "adir" / "bdirlink").symlink_to("../bdir")
     inwd.add_and_commit()
     assert set(find_files("adir")) == _sep({"adir/filea", "adir/bdirlink/fileb"})
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(sys.platform == "win32", reason="symlinks to dir not supported")
 def test_symlink_dir_source_not_in_scm(inwd: WorkDir) -> None:
     (inwd.cwd / "adir" / "bdirlink").symlink_to("../bdir")
     assert set(find_files("adir")) == _sep({"adir/filea"})
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(
     sys.platform == "win32", reason="symlinks to files not supported on windows"
 )
@@ -108,6 +115,7 @@ def test_symlink_file(inwd: WorkDir) -> None:
     )  # -> ../file1
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(
     sys.platform == "win32", reason="symlinks to files not supported on windows"
 )
@@ -116,13 +124,15 @@ def test_symlink_file_source_not_in_scm(inwd: WorkDir) -> None:
     assert set(find_files("adir")) == _sep({"adir/filea"})
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(sys.platform == "win32", reason="symlinks to dir not supported")
 def test_symlink_loop(inwd: WorkDir) -> None:
     (inwd.cwd / "adir" / "loop").symlink_to("../adir")
     inwd.add_and_commit()
     assert set(find_files("adir")) == _sep({"adir/filea", "adir/loop"})  # -> ../adir
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(sys.platform == "win32", reason="symlinks to dir not supported")
 def test_symlink_loop_outside_path(inwd: WorkDir) -> None:
     (inwd.cwd / "bdir" / "loop").symlink_to("../bdir")
@@ -131,13 +141,15 @@ def test_symlink_loop_outside_path(inwd: WorkDir) -> None:
     assert set(find_files("adir")) == _sep({"adir/filea", "adir/bdirlink/fileb"})
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(sys.platform == "win32", reason="symlinks to dir not supported")
 def test_symlink_dir_out_of_git(inwd: WorkDir) -> None:
     (inwd.cwd / "adir" / "outsidedirlink").symlink_to(os.path.join(__file__, ".."))
     inwd.add_and_commit()
     assert set(find_files("adir")) == _sep({"adir/filea"})
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(
     sys.platform == "win32", reason="symlinks to files not supported on windows"
 )
@@ -147,6 +159,7 @@ def test_symlink_file_out_of_git(inwd: WorkDir) -> None:
     assert set(find_files("adir")) == _sep({"adir/filea"})
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.parametrize("path_add", ["{cwd}", "{cwd}" + os.pathsep + "broken"])
 def test_ignore_root(
     inwd: WorkDir, monkeypatch: pytest.MonkeyPatch, path_add: str
@@ -155,6 +168,7 @@ def test_ignore_root(
     assert find_files() == []
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 def test_empty_root(inwd: WorkDir) -> None:
     subdir = inwd.cwd / "cdir" / "subdir"
     subdir.mkdir(parents=True)
@@ -163,6 +177,7 @@ def test_empty_root(inwd: WorkDir) -> None:
     assert set(find_files("cdir")) == _sep({"cdir/subdir/filec"})
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 def test_empty_subdir(inwd: WorkDir) -> None:
     subdir = inwd.cwd / "adir" / "emptysubdir" / "subdir"
     subdir.mkdir(parents=True)
@@ -173,6 +188,7 @@ def test_empty_subdir(inwd: WorkDir) -> None:
     )
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(sys.platform == "win32", reason="symlinks not supported on windows")
 def test_double_include_through_symlink(inwd: WorkDir) -> None:
     (inwd.cwd / "data").mkdir()
@@ -192,6 +208,7 @@ def test_double_include_through_symlink(inwd: WorkDir) -> None:
     )
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.skipif(sys.platform == "win32", reason="symlinks not supported on windows")
 def test_symlink_not_in_scm_while_target_is(inwd: WorkDir) -> None:
     (inwd.cwd / "data").mkdir()
@@ -211,12 +228,14 @@ def test_symlink_not_in_scm_while_target_is(inwd: WorkDir) -> None:
     )
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.issue(587)
 @pytest.mark.skip_commit
 def test_not_commited(inwd: WorkDir) -> None:
     assert find_files() == []
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 def test_unexpanded_git_archival(wd: WorkDir, monkeypatch: pytest.MonkeyPatch) -> None:
     # When substitutions in `.git_archival.txt` are not expanded, files should
     # not be automatically listed.
@@ -226,6 +245,7 @@ def test_unexpanded_git_archival(wd: WorkDir, monkeypatch: pytest.MonkeyPatch) -
     assert find_files() == []
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 @pytest.mark.parametrize("archive_file", [".git_archival.txt", ".hg_archival.txt"])
 def test_archive(
     wd: WorkDir, monkeypatch: pytest.MonkeyPatch, archive_file: str

testing/test_git.py

@@ -5,6 +5,7 @@
 import shutil
 import subprocess
 import sys
+import textwrap
 
 from datetime import date
 from datetime import datetime
@@ -19,6 +20,7 @@
 import pytest
 
 import setuptools_scm._file_finders
+import setuptools_scm._file_finders.git
 
 from setuptools_scm import Configuration
 from setuptools_scm import NonNormalizedVersion
@@ -117,11 +119,10 @@ def test_git_gone(wd: WorkDir, monkeypatch: pytest.MonkeyPatch) -> None:
 @pytest.mark.issue("https://github.com/pypa/setuptools-scm/issues/298")
 @pytest.mark.issue(403)
 def test_file_finder_no_history(wd: WorkDir, caplog: pytest.LogCaptureFixture) -> None:
-    file_list = git_find_files(str(wd.cwd))
+    with pytest.warns(UserWarning, match=r"pretending there aren't any"):
+        file_list = git_find_files(str(wd.cwd))
     assert file_list == []
 
-    assert "listing git files failed - pretending there aren't any" in caplog.text
-
 
 @pytest.mark.issue("https://github.com/pypa/setuptools-scm/issues/281")
 def test_parse_call_order(wd: WorkDir) -> None:
@@ -160,7 +161,8 @@ def break_folder_permissions(path: Path) -> Generator[None, None, None]:
         sudo_devnull(["chgrp", "-R", str(original_stat.st_gid), path], check=True)
 
 
-@pytest.mark.issue("https://github.com/pypa/setuptools-scm/issues/707")
+@pytest.mark.filterwarnings("ignore:listing git files failed")
+@pytest.mark.issue("https://github.com/pypa/setuptools-scm/issues/784")
 def test_not_owner(wd: WorkDir) -> None:
     with break_folder_permissions(wd.cwd):
         assert git.parse(str(wd.cwd), Configuration())
@@ -861,3 +863,37 @@ def test_git_no_commits_uses_fallback_version(wd: WorkDir) -> None:
     assert str(version_no_fallback.tag) == "0.0"
     assert version_no_fallback.distance == 0
     assert version_no_fallback.dirty is True
+
+
+@pytest.mark.issue("https://github.com/pypa/setuptools-scm/issues/707")
+def test_dubious_dir(
+    wd: WorkDir, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Test that we exit clearly if we are in a unsafe directory"""
+    wd.commit_testfile()
+    git_wd = git.GitWorkdir(wd.cwd)
+
+    # This fakes "git rev-parse HEAD" as if it were in a directory not
+    # owned by you.
+    stderr = textwrap.dedent(f"""
+    fatal: detected dubious ownership in repository at '{git_wd}'
+    To add an exception for this directory, call:
+
+    	git config --global --add safe.directory /this/is/a/fake/path
+    """)
+    orig_run = run
+
+    def _run(*args, **kwargs) -> CompletedProcess:  # type: ignore[no-untyped-def]
+        if args[0] == ["git", "rev-parse", "HEAD"]:
+            return CompletedProcess(
+                args=[], stdout="%cI", stderr=stderr, returncode=128
+            )
+        return orig_run(*args, **kwargs)
+
+    with pytest.raises(SystemExit):
+        with patch.object(setuptools_scm._file_finders.git, "_run", _run):
+            git_find_files(str(wd.cwd))
+
+    assert "fatal: detected dubious ownership in repository" in " ".join(
+        caplog.messages
+    )

testing/test_mercurial.py

@@ -97,6 +97,7 @@ def test_hg_command_from_env_is_invalid(
         assert wd.get_version(fallback_version="1.0") == "1.0"
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 def test_find_files_stop_at_root_hg(
     wd: WorkDir, monkeypatch: pytest.MonkeyPatch
 ) -> None:

testing/test_regressions.py

@@ -148,6 +148,7 @@ def test_case_mismatch_nested_dir_windows_git(tmp_path: Path) -> None:
         raise
 
 
+@pytest.mark.filterwarnings("ignore:listing git files failed")
 def test_case_mismatch_force_assertion_failure(tmp_path: Path) -> None:
     """Force the assertion failure by directly calling _git_toplevel with mismatched paths"""
     from setuptools_scm._file_finders.git import _git_toplevel