Better diagnostics for people using nix subshell on non-NixOS.

1. Turned patch-binaries-for-nix from a boolean into a ternary flag: true,
   false, and unset.

2. When patch-binaries-for-nix is unset, we continue with the existing NixOS
   detection heuristic (look for nixos in /etc/os-release, if present), but if
   we are not atop NixOS, then issue a note if we see the IN_NIX_SHELL
   environment variable telling the user to consider setting
   patch-binaries-for-nix explicitly.

Author: pnkfelix

src/bootstrap/bootstrap.py

@@ -643,18 +643,33 @@ def get_answer():
             if ostype != "Linux":
                 return False
 
-            # If the user has asked binaries to be patched for Nix, then
-            # don't check for NixOS.
+            # If the user has explicitly indicated whether binaries should be
+            # patched for Nix, then don't check for NixOS.
             if self.get_toml("patch-binaries-for-nix", "build") == "true":
                 return True
+            if self.get_toml("patch-binaries-for-nix", "build") == "false":
+                return False
+
+            # Assume we should fix until we see evidence that it is not NixOS
+            should_fix_retval = True
 
             # Use `/etc/os-release` instead of `/etc/NIXOS`.
             # The latter one does not exist on NixOS when using tmpfs as root.
             try:
                 with open("/etc/os-release", "r") as f:
-                    return any(ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"') for ln in f)
+                    should_fix_retval = any(ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"') for ln in f)
             except FileNotFoundError:
-                return False
+                should_fix_retval = False
+
+            # If not on NixOS, then warn if user seems to be atop Nix shell
+            if not should_fix_retval:
+                in_nix_shell = os.getenv('IN_NIX_SHELL')
+                if in_nix_shell:
+                    print("The IN_NIX_SHELL environment variable is set to `{}`;".format(in_nix_shell),
+                          "you may need to set `patch-binaries-for-nix=true` in your config.toml",
+                          file=sys.stderr)
+
+            return should_fix_retval
 
         answer = self._should_fix_bins_and_dylibs = get_answer()
         if answer:

src/bootstrap/config.rs

@@ -137,7 +137,7 @@ pub struct Config {
     pub json_output: bool,
     pub test_compare_mode: bool,
     pub color: Color,
-    pub patch_binaries_for_nix: bool,
+    pub patch_binaries_for_nix: Option<bool>,
     pub stage0_metadata: Stage0Metadata,
 
     pub stdout_is_tty: bool,
@@ -1338,7 +1338,7 @@ impl Config {
         set(&mut config.local_rebuild, build.local_rebuild);
         set(&mut config.print_step_timings, build.print_step_timings);
         set(&mut config.print_step_rusage, build.print_step_rusage);
-        set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix);
+        config.patch_binaries_for_nix = build.patch_binaries_for_nix;
 
         config.verbose = cmp::max(config.verbose, flags.verbose as usize);
 

src/bootstrap/download.rs

@@ -91,8 +91,8 @@ impl Config {
             // NOTE: this intentionally comes after the Linux check:
             // - patchelf only works with ELF files, so no need to run it on Mac or Windows
             // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
-            if self.patch_binaries_for_nix {
-                return true;
+            if let Some(explicit_value) = self.patch_binaries_for_nix {
+                return explicit_value;
             }
 
             // Use `/etc/os-release` instead of `/etc/NIXOS`.
@@ -105,6 +105,15 @@ impl Config {
                     matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"")
                 }),
             };
+            if !is_nixos {
+                let in_nix_shell = env::var("IN_NIX_SHELL");
+                if let Ok(in_nix_shell) = in_nix_shell {
+                    eprintln!(
+                        "The IN_NIX_SHELL environment variable is set to `{in_nix_shell}`; \
+                         you may need to set `patch-binaries-for-nix=true` in your config.toml"
+                    );
+                }
+            }
             is_nixos
         });
         if val {