Auto merge of #42081 - ishitatsuyuki:submodule-better, r=aidanhs

Use the improved submodule handling

r? @alexcrichton

That was a crap...
```
Updating submodules
Traceback (most recent call last):
  File "./x.py", line 20, in <module>
    bootstrap.main()
  File "/home/ishitatsuyuki/Documents/rust/src/bootstrap/bootstrap.py", line 684, in main
    bootstrap()
  File "/home/ishitatsuyuki/Documents/rust/src/bootstrap/bootstrap.py", line 662, in bootstrap
    rb.update_submodules()
  File "/home/ishitatsuyuki/Documents/rust/src/bootstrap/bootstrap.py", line 566, in update_submodules
    path = line[1:].split(' ')[1]
TypeError: a bytes-like object is required, not 'str'
```

Maybe we need to confirm the compatibility of git options, such as `git config` or `git -C` (I believe they existed long before, though). This is tested locally.

Author: bors

src/bootstrap/bootstrap.py

@@ -40,7 +40,8 @@ def get(url, path, verbose=False):
                 return
             else:
                 if verbose:
-                    print("ignoring already-download file " + path + " due to failed verification")
+                    print("ignoring already-download file " +
+                          path + " due to failed verification")
                 os.unlink(path)
         download(temp_path, url, True, verbose)
         if not verify(temp_path, sha_path, verbose):
@@ -100,8 +101,8 @@ def verify(path, sha_path, verbose):
     verified = found == expected
     if not verified:
         print("invalid checksum:\n"
-               "    found:    {}\n"
-               "    expected: {}".format(found, expected))
+              "    found:    {}\n"
+              "    expected: {}".format(found, expected))
     return verified
 
 
@@ -127,20 +128,21 @@ def unpack(tarball, dst, verbose=False, match=None):
             shutil.move(tp, fp)
     shutil.rmtree(os.path.join(dst, fname))
 
-def run(args, verbose=False, exception=False, cwd=None, env=None):
+def run(args, verbose=False, exception=False, **kwargs):
     if verbose:
         print("running: " + ' '.join(args))
     sys.stdout.flush()
     # Use Popen here instead of call() as it apparently allows powershell on
     # Windows to not lock up waiting for input presumably.
-    ret = subprocess.Popen(args, cwd=cwd, env=env)
+    ret = subprocess.Popen(args, **kwargs)
     code = ret.wait()
     if code != 0:
         err = "failed to run: " + ' '.join(args)
         if verbose or exception:
             raise RuntimeError(err)
         sys.exit(err)
 
+
 def stage0_data(rust_root):
     nightlies = os.path.join(rust_root, "src/stage0.txt")
     data = {}
@@ -153,11 +155,13 @@ def stage0_data(rust_root):
             data[a] = b
     return data
 
+
 def format_build_time(duration):
     return str(datetime.timedelta(seconds=int(duration)))
 
 
 class RustBuild(object):
+
     def download_stage0(self):
         cache_dst = os.path.join(self.build_dir, "cache")
         rustc_cache = os.path.join(cache_dst, self.stage0_date())
@@ -172,11 +176,13 @@ def download_stage0(self):
             self.print_what_it_means_to_bootstrap()
             if os.path.exists(self.bin_root()):
                 shutil.rmtree(self.bin_root())
-            filename = "rust-std-{}-{}.tar.gz".format(rustc_channel, self.build)
+            filename = "rust-std-{}-{}.tar.gz".format(
+                rustc_channel, self.build)
             url = self._download_url + "/dist/" + self.stage0_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
-                get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
+                get("{}/{}".format(url, filename),
+                    tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(),
                    match="rust-std-" + self.build,
                    verbose=self.verbose)
@@ -185,20 +191,25 @@ def download_stage0(self):
             url = self._download_url + "/dist/" + self.stage0_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
-                get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
-            unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
+                get("{}/{}".format(url, filename),
+                    tarball, verbose=self.verbose)
+            unpack(tarball, self.bin_root(),
+                   match="rustc", verbose=self.verbose)
             self.fix_executable(self.bin_root() + "/bin/rustc")
             self.fix_executable(self.bin_root() + "/bin/rustdoc")
             with open(self.rustc_stamp(), 'w') as f:
                 f.write(self.stage0_date())
 
             if "pc-windows-gnu" in self.build:
-                filename = "rust-mingw-{}-{}.tar.gz".format(rustc_channel, self.build)
+                filename = "rust-mingw-{}-{}.tar.gz".format(
+                    rustc_channel, self.build)
                 url = self._download_url + "/dist/" + self.stage0_date()
                 tarball = os.path.join(rustc_cache, filename)
                 if not os.path.exists(tarball):
-                    get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
-                unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
+                    get("{}/{}".format(url, filename),
+                        tarball, verbose=self.verbose)
+                unpack(tarball, self.bin_root(),
+                       match="rust-mingw", verbose=self.verbose)
 
         if self.cargo().startswith(self.bin_root()) and \
                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
@@ -207,8 +218,10 @@ def download_stage0(self):
             url = self._download_url + "/dist/" + self.stage0_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
-                get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
-            unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
+                get("{}/{}".format(url, filename),
+                    tarball, verbose=self.verbose)
+            unpack(tarball, self.bin_root(),
+                   match="cargo", verbose=self.verbose)
             self.fix_executable(self.bin_root() + "/bin/cargo")
             with open(self.cargo_stamp(), 'w') as f:
                 f.write(self.stage0_date())
@@ -218,7 +231,8 @@ def fix_executable(self, fname):
 
         default_encoding = sys.getdefaultencoding()
         try:
-            ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
+            ostype = subprocess.check_output(
+                ['uname', '-s']).strip().decode(default_encoding)
         except (subprocess.CalledProcessError, WindowsError):
             return
 
@@ -234,7 +248,8 @@ def fix_executable(self, fname):
         print("info: you seem to be running NixOS. Attempting to patch " + fname)
 
         try:
-            interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
+            interpreter = subprocess.check_output(
+                ["patchelf", "--print-interpreter", fname])
             interpreter = interpreter.strip().decode(default_encoding)
         except subprocess.CalledProcessError as e:
             print("warning: failed to call patchelf: %s" % e)
@@ -243,7 +258,8 @@ def fix_executable(self, fname):
         loader = interpreter.split("/")[-1]
 
         try:
-            ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
+            ldd_output = subprocess.check_output(
+                ['ldd', '/run/current-system/sw/bin/sh'])
             ldd_output = ldd_output.strip().decode(default_encoding)
         except subprocess.CalledProcessError as e:
             print("warning: unable to call ldd: %s" % e)
@@ -261,7 +277,8 @@ def fix_executable(self, fname):
         correct_interpreter = loader_path + loader
 
         try:
-            subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
+            subprocess.check_output(
+                ["patchelf", "--set-interpreter", correct_interpreter, fname])
         except subprocess.CalledProcessError as e:
             print("warning: failed to call patchelf: %s" % e)
             return
@@ -371,16 +388,16 @@ def build_bootstrap(self):
         env["CARGO_TARGET_DIR"] = build_dir
         env["RUSTC"] = self.rustc()
         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
-                                 (os.pathsep + env["LD_LIBRARY_PATH"]) \
-                                 if "LD_LIBRARY_PATH" in env else ""
+            (os.pathsep + env["LD_LIBRARY_PATH"]) \
+            if "LD_LIBRARY_PATH" in env else ""
         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
-                                   (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
-                                   if "DYLD_LIBRARY_PATH" in env else ""
+            (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
+            if "DYLD_LIBRARY_PATH" in env else ""
         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
-                                   (os.pathsep + env["LIBRARY_PATH"]) \
-                                   if "LIBRARY_PATH" in env else ""
+            (os.pathsep + env["LIBRARY_PATH"]) \
+            if "LIBRARY_PATH" in env else ""
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
-                      os.pathsep + env["PATH"]
+            os.pathsep + env["PATH"]
         if not os.path.isfile(self.cargo()):
             raise Exception("no cargo executable found at `%s`" % self.cargo())
         args = [self.cargo(), "build", "--manifest-path",
@@ -395,16 +412,6 @@ def build_bootstrap(self):
             args.append("--frozen")
         run(args, env=env, verbose=self.verbose)
 
-    def output(self, args, env=None, cwd=None):
-        default_encoding = sys.getdefaultencoding()
-        proc = subprocess.Popen(args, stdout=subprocess.PIPE, env=env, cwd=cwd)
-        (out, err) = proc.communicate()
-        ret = proc.wait()
-        if ret != 0:
-            print(out)
-            sys.exit(ret)
-        return out.decode(default_encoding)
-
     def build_triple(self):
         default_encoding = sys.getdefaultencoding()
         config = self.get_toml('build')
@@ -414,8 +421,10 @@ def build_triple(self):
         if config:
             return config
         try:
-            ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
-            cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
+            ostype = subprocess.check_output(
+                ['uname', '-s']).strip().decode(default_encoding)
+            cputype = subprocess.check_output(
+                ['uname', '-m']).strip().decode(default_encoding)
         except (subprocess.CalledProcessError, OSError):
             if sys.platform == 'win32':
                 return 'x86_64-pc-windows-msvc'
@@ -427,7 +436,8 @@ def build_triple(self):
         # The goal here is to come up with the same triple as LLVM would,
         # at least for the subset of platforms we're willing to target.
         if ostype == 'Linux':
-            os_from_sp = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding)
+            os_from_sp = subprocess.check_output(
+                ['uname', '-o']).strip().decode(default_encoding)
             if os_from_sp == 'Android':
                 ostype = 'linux-android'
             else:
@@ -451,7 +461,7 @@ def build_triple(self):
             # must be used instead.
             try:
                 cputype = subprocess.check_output(['isainfo',
-                  '-k']).strip().decode(default_encoding)
+                                                   '-k']).strip().decode(default_encoding)
             except (subprocess.CalledProcessError, OSError):
                 err = "isainfo not found"
                 if self.verbose:
@@ -544,51 +554,29 @@ def build_triple(self):
 
     def update_submodules(self):
         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
-            self.get_toml('submodules') == "false" or \
-            self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
+                self.get_toml('submodules') == "false" or \
+                self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
             return
-
         print('Updating submodules')
-        output = self.output(["git", "submodule", "status"], cwd=self.rust_root)
-        submodules = []
-        for line in output.splitlines():
-            # NOTE `git submodule status` output looks like this:
-            #
-            # -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc
-            # +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..)
-            #  e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6)
-            #
-            # The first character can be '-', '+' or ' ' and denotes the
-            # `State` of the submodule Right next to this character is the
-            # SHA-1 of the submodule HEAD And after that comes the path to the
-            # submodule
-            path = line[1:].split(' ')[1]
-            submodules.append([path, line[0]])
-
-        run(["git", "submodule", "sync"], cwd=self.rust_root)
-
-        for submod in submodules:
-            path, status = submod
-            if path.endswith('llvm') and \
-                (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT')):
-                continue
-            if path.endswith('jemalloc') and \
-                (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')):
-                continue
-            submod_path = os.path.join(self.rust_root, path)
-
-            if status == ' ':
-                run(["git", "reset", "--hard"], cwd=submod_path)
-                run(["git", "clean", "-fdx"], cwd=submod_path)
-            elif status == '+':
-                run(["git", "submodule", "update", path], cwd=self.rust_root)
-                run(["git", "reset", "--hard"], cwd=submod_path)
-                run(["git", "clean", "-fdx"], cwd=submod_path)
-            elif status == '-':
-                run(["git", "submodule", "init", path], cwd=self.rust_root)
-                run(["git", "submodule", "update", path], cwd=self.rust_root)
-            else:
-                raise ValueError('unknown submodule status: ' + status)
+        default_encoding = sys.getdefaultencoding()
+        run(["git", "submodule", "-q", "sync"], cwd=self.rust_root)
+        submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
+            ["git", "config", "--file", os.path.join(self.rust_root, ".gitmodules"),
+             "--get-regexp", "path"]
+        ).decode(default_encoding).splitlines()]
+        submodules = [module for module in submodules
+                      if not ((module.endswith("llvm") and
+                               (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT'))) or
+                              (module.endswith("jemalloc") and
+                               (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT'))))
+                     ]
+        run(["git", "submodule", "update",
+                  "--init"] + submodules, cwd=self.rust_root, verbose=self.verbose)
+        run(["git", "submodule", "-q", "foreach", "git",
+                  "reset", "-q", "--hard"], cwd=self.rust_root, verbose=self.verbose)
+        run(["git", "submodule", "-q", "foreach", "git",
+                  "clean", "-qdfx"], cwd=self.rust_root, verbose=self.verbose)
+
 
 def bootstrap():
     parser = argparse.ArgumentParser(description='Build rust')
@@ -641,7 +629,7 @@ def bootstrap():
     if rb.use_vendored_sources:
         if not os.path.exists('.cargo'):
             os.makedirs('.cargo')
-        with open('.cargo/config','w') as f:
+        with open('.cargo/config', 'w') as f:
             f.write("""
                 [source.crates-io]
                 replace-with = 'vendored-sources'
@@ -681,21 +669,25 @@ def bootstrap():
     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
     run(args, env=env, verbose=rb.verbose)
 
+
 def main():
     start_time = time()
-    help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
+    help_triggered = (
+        '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
     try:
         bootstrap()
         if not help_triggered:
-            print("Build completed successfully in %s" % format_build_time(time() - start_time))
+            print("Build completed successfully in %s" %
+                  format_build_time(time() - start_time))
     except (SystemExit, KeyboardInterrupt) as e:
         if hasattr(e, 'code') and isinstance(e.code, int):
             exit_code = e.code
         else:
             exit_code = 1
             print(e)
         if not help_triggered:
-            print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
+            print("Build completed unsuccessfully in %s" %
+                  format_build_time(time() - start_time))
         sys.exit(exit_code)
 
 if __name__ == '__main__':