Auto merge of rust-lang#111067 - albertlarsan68:fix-multiprocessing-x-py, r=jyn514

Make x.py work again in most (all?) cases

Fixes rust-lang#111046.

Wrap all of x.py in `if __name__ == '__main__':` to avoid problems with `multiprocessing`
Make the pool sizing better

Author: bors

src/bootstrap/bootstrap.py

@@ -28,6 +28,20 @@ def platform_is_win32():
 else:
     EXE_SUFFIX = ""
 
+def get_cpus():
+    if hasattr(os, "sched_getaffinity"):
+        return len(os.sched_getaffinity(0))
+    if hasattr(os, "cpu_count"):
+        cpus = os.cpu_count()
+        if cpus is not None:
+            return cpus
+    try:
+        return cpu_count()
+    except NotImplementedError:
+        return 1
+
+
+
 def get(base, url, path, checksums, verbose=False):
     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
         temp_path = temp_file.name
@@ -540,11 +554,15 @@ def download_toolchain(self):
 
             # Unpack the tarballs in parallle.
             # In Python 2.7, Pool cannot be used as a context manager.
-            p = Pool(min(len(tarballs_download_info), cpu_count()))
+            pool_size = min(len(tarballs_download_info), get_cpus())
+            if self.verbose:
+                print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs')
+            p = Pool(pool_size)
             try:
                 p.map(unpack_component, tarballs_download_info)
             finally:
                 p.close()
+            p.join()
 
             if self.should_fix_bins_and_dylibs():
                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))

x.py

@@ -4,26 +4,29 @@
 
 # This file is only a "symlink" to bootstrap.py, all logic should go there.
 
-import os
-import sys
+# Parts of `bootstrap.py` use the `multiprocessing` module, so this entry point
+# must use the normal `if __name__ == '__main__':` convention to avoid problems.
+if __name__ == '__main__':
+    import os
+    import sys
 
-# If this is python2, check if python3 is available and re-execute with that
-# interpreter. Only python3 allows downloading CI LLVM.
-#
-# This matters if someone's system `python` is python2.
-if sys.version_info.major < 3:
-    try:
-        os.execvp("py", ["py", "-3"] + sys.argv)
-    except OSError:
+    # If this is python2, check if python3 is available and re-execute with that
+    # interpreter. Only python3 allows downloading CI LLVM.
+    #
+    # This matters if someone's system `python` is python2.
+    if sys.version_info.major < 3:
         try:
-            os.execvp("python3", ["python3"] + sys.argv)
+            os.execvp("py", ["py", "-3"] + sys.argv)
         except OSError:
-            # Python 3 isn't available, fall back to python 2
-            pass
+            try:
+                os.execvp("python3", ["python3"] + sys.argv)
+            except OSError:
+                # Python 3 isn't available, fall back to python 2
+                pass
 
-rust_dir = os.path.dirname(os.path.abspath(__file__))
-# For the import below, have Python search in src/bootstrap first.
-sys.path.insert(0, os.path.join(rust_dir, "src", "bootstrap"))
+    rust_dir = os.path.dirname(os.path.abspath(__file__))
+    # For the import below, have Python search in src/bootstrap first.
+    sys.path.insert(0, os.path.join(rust_dir, "src", "bootstrap"))
 
-import bootstrap
-bootstrap.main()
+    import bootstrap
+    bootstrap.main()