[utils] Force subprocess output to be read in text mode

In Python 3, subprocess output is read as binary data by default, which isn’t what we want.

Instead of reading process output as byte strings, then manually decoding them into strings, simply pass `text=True` to functions in the `subprocess` module, so that we get properly decoded strings right out the box.

This fixes places that forget to do the decoding step — most especially, the `update-checkout` script. That script prints Git output as byte strings, which leads to unreadable results.

Additionally, in shell.run, use the same pipe for capturing both stdout and stderr. The distinction is pretty pointless in this use case; however, keeping the two channels separate means that we lose the original ordering of printed messages, which does matter.

Author: lorentey

utils/refactor-check-compiles.py

@@ -9,7 +9,7 @@
 
 def run_cmd(cmd, desc):
     try:
-        return subprocess.check_output(cmd)
+        return subprocess.check_output(cmd, text=True)
     except subprocess.CalledProcessError:
         print('FAILED ' + desc + ':', file=sys.stderr)
         print(' '.join(cmd), file=sys.stderr)
@@ -130,7 +130,7 @@ def main():
         '-source-filename', args.source_filename,
         '-rewritten-output-file', temp_file_path,
         '-pos', args.pos
-    ] + extra_refactor_args + extra_both_args, desc='producing edit').decode("utf-8")
+    ] + extra_refactor_args + extra_both_args, desc='producing edit')
     sys.stdout.write(dump_text_output)
 
     run_cmd([

utils/swift-darwin-postprocess.py

@@ -32,7 +32,8 @@ def main(arguments):
 # (rdar://78851265)
 def unrpathize(filename):
     dylibsOutput = subprocess.check_output(
-        ['xcrun', 'dyldinfo', '-dylibs', filename])
+        ['xcrun', 'dyldinfo', '-dylibs', filename],
+        text=True)
 
     # Do not rewrite @rpath-relative load commands for these libraries:
     # they are test support libraries that are never installed under
@@ -60,8 +61,7 @@ def unrpathize(filename):
 
     # Build a command to invoke install_name_tool.
     command = ['install_name_tool']
-    for binaryline in dylibsOutput.splitlines():
-        line = binaryline.decode("utf-8", "strict")
+    for line in dylibsOutput.splitlines():
         match = dylib_regex.match(line)
         if match and match.group('filename') not in allow_list:
             command.append('-change')

utils/swift-rpathize.py

@@ -55,7 +55,8 @@ def main(arguments):
 
 def rpathize(filename):
     dylibsOutput = subprocess.check_output(
-        ['xcrun', 'dyldinfo', '-dylibs', filename])
+        ['xcrun', 'dyldinfo', '-dylibs', filename],
+        text=True)
 
     # The output from dyldinfo -dylibs is a line of header followed by one
     # install name per line, indented with spaces.
@@ -64,8 +65,7 @@ def rpathize(filename):
 
     # Build a command to invoke install_name_tool.
     command = ['install_name_tool']
-    for binaryline in dylibsOutput.splitlines():
-        line = binaryline.decode("utf-8", "strict")
+    for line in dylibsOutput.splitlines():
         match = dylib_regex.match(line)
         if match:
             command.append('-change')

utils/swift_build_support/swift_build_support/shell.py

@@ -132,12 +132,11 @@ def capture(command, stderr=None, env=None, dry_run=None, echo=True,
         _env = dict(os.environ)
         _env.update(env)
     try:
-        out = subprocess.check_output(command, env=_env, stderr=stderr)
-        # Coerce to `str` hack. not py3 `byte`, not py2 `unicode`.
-        return str(out.decode())
+        return subprocess.check_output(command, env=_env, stderr=stderr,
+                                       text=True)
     except subprocess.CalledProcessError as e:
         if allow_non_zero_exit:
-            return str(e.output.decode())
+            return e.output
         if optional:
             return None
         _fatal_error(
@@ -218,19 +217,18 @@ def run(*args, **kwargs):
         return(None, 0, args)
 
     my_pipe = subprocess.Popen(
-        *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
-    (stdout, stderr) = my_pipe.communicate()
+        *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
+        **kwargs)
+    (output, _) = my_pipe.communicate()
     ret = my_pipe.wait()
 
     if lock:
         lock.acquire()
     if echo_output:
         print(repo_path)
         _echo_command(dry_run, *args, env=env)
-        if stdout:
-            print(stdout, end="")
-        if stderr:
-            print(stderr, end="")
+        if output:
+            print(output, end="")
         print()
     if lock:
         lock.release()
@@ -240,6 +238,6 @@ def run(*args, **kwargs):
         eout.ret = ret
         eout.args = args
         eout.repo_path = repo_path
-        eout.stderr = stderr
+        eout.stderr = output
         raise eout
-    return (stdout, 0, args)
+    return (output, 0, args)

utils/update_checkout/update_checkout/update_checkout.py

@@ -70,7 +70,7 @@ def check_parallel_results(results, op):
             print("%s failed (ret=%d): %s" % (r.repo_path, r.ret, r))
             fail_count += 1
             if r.stderr:
-                print(r.stderr.decode('utf-8'))
+                print(r.stderr)
     return fail_count