Move some low level utility code to utils.py. NFC (#25548)

Author: sbc100

emcc.py

@@ -35,7 +35,6 @@
 
 from tools import shared, system_libs, utils, cmdline
 from tools import diagnostics, building, compile
-from tools.shared import unsuffixed_basename, get_file_suffix
 from tools.shared import exit_with_error, DEBUG
 from tools.shared import in_temp
 from tools.shared import DYLIB_EXTENSIONS
@@ -44,7 +43,7 @@
 from tools import config
 from tools import cache
 from tools.settings import default_setting, user_settings, settings, COMPILE_TIME_SETTINGS
-from tools.utils import read_file
+from tools.utils import read_file, unsuffixed_basename, get_file_suffix
 
 logger = logging.getLogger('emcc')
 
@@ -566,7 +565,7 @@ def compile_source_file(input_file):
     if not shared.SKIP_SUBPROCS:
       assert os.path.exists(output_file)
       if options.save_temps:
-        shutil.copyfile(output_file, shared.unsuffixed_basename(input_file) + '.o')
+        shutil.copyfile(output_file, utils.unsuffixed_basename(input_file) + '.o')
     return output_file
 
   # Compile input files individually to temporary locations.

test/common.py

@@ -38,7 +38,7 @@
 import clang_native
 import jsrun
 import line_endings
-from tools.shared import EMCC, EMXX, DEBUG, exe_suffix
+from tools.shared import EMCC, EMXX, DEBUG
 from tools.shared import get_canonical_temp_dir, path_from_root
 from tools.utils import MACOS, WINDOWS, read_file, read_binary, write_binary, exit_with_error
 from tools.settings import COMPILE_TIME_SETTINGS
@@ -126,7 +126,7 @@ class FirefoxConfig:
   data_dir_flag = '-profile '
   default_flags = ('-new-instance',)
   headless_flags = '-headless'
-  executable_name = exe_suffix('firefox')
+  executable_name = utils.exe_suffix('firefox')
 
   @staticmethod
   def configure(data_dir):
@@ -158,13 +158,13 @@ def configure(data_dir):
 
 DEFAULT_BROWSER_DATA_DIR = path_from_root('out/browser-profile')
 
-WEBIDL_BINDER = shared.bat_suffix(path_from_root('tools/webidl_binder'))
+WEBIDL_BINDER = utils.bat_suffix(path_from_root('tools/webidl_binder'))
 
-EMBUILDER = shared.bat_suffix(path_from_root('embuilder'))
-EMMAKE = shared.bat_suffix(path_from_root('emmake'))
-EMCMAKE = shared.bat_suffix(path_from_root('emcmake'))
-EMCONFIGURE = shared.bat_suffix(path_from_root('emconfigure'))
-EMRUN = shared.bat_suffix(shared.path_from_root('emrun'))
+EMBUILDER = utils.bat_suffix(path_from_root('embuilder'))
+EMMAKE = utils.bat_suffix(path_from_root('emmake'))
+EMCMAKE = utils.bat_suffix(path_from_root('emcmake'))
+EMCONFIGURE = utils.bat_suffix(path_from_root('emconfigure'))
+EMRUN = utils.bat_suffix(shared.path_from_root('emrun'))
 WASM_DIS = os.path.join(building.get_binaryen_bin(), 'wasm-dis')
 LLVM_OBJDUMP = shared.llvm_tool_path('llvm-objdump')
 PYTHON = sys.executable
@@ -241,7 +241,7 @@ def get_browser_config():
 
 
 def compiler_for(filename, force_c=False):
-  if shared.suffix(filename) in ('.cc', '.cxx', '.cpp') and not force_c:
+  if utils.suffix(filename) in ('.cc', '.cxx', '.cpp') and not force_c:
     return EMXX
   else:
     return EMCC
@@ -1532,7 +1532,7 @@ def build(self, filename, libraries=None, includes=None, force_c=False, cflags=N
     compiler = [compiler_for(filename, force_c)]
 
     if force_c:
-      assert shared.suffix(filename) != '.c', 'force_c is not needed for source files ending in .c'
+      assert utils.suffix(filename) != '.c', 'force_c is not needed for source files ending in .c'
       compiler.append('-xc')
 
     all_cflags = self.get_cflags(main_file=True)
@@ -1544,7 +1544,7 @@ def build(self, filename, libraries=None, includes=None, force_c=False, cflags=N
     if output_basename:
       output = output_basename + output_suffix
     else:
-      output = shared.unsuffixed_basename(filename) + output_suffix
+      output = utils.unsuffixed_basename(filename) + output_suffix
     cmd = compiler + [str(filename), '-o', output] + all_cflags
     if libraries:
       cmd += libraries
@@ -1984,7 +1984,7 @@ def _test_dylink_dso_needed(self, do_run):
     so = '.wasm' if self.is_wasm() else '.js'
 
     def ccshared(src, linkto=None):
-      cmdv = [EMCC, src, '-o', shared.unsuffixed(src) + so, '-sSIDE_MODULE'] + self.get_cflags()
+      cmdv = [EMCC, src, '-o', utils.unsuffixed(src) + so, '-sSIDE_MODULE'] + self.get_cflags()
       if linkto:
         cmdv += linkto
       self.run_process(cmdv)
@@ -2060,7 +2060,7 @@ def do_runf(self, filename, expected_output=None, **kwargs):
   def do_run_in_out_file_test(self, srcfile, **kwargs):
     srcfile = maybe_test_file(srcfile)
     out_suffix = kwargs.pop('out_suffix', '')
-    outfile = shared.unsuffixed(srcfile) + out_suffix + '.out'
+    outfile = utils.unsuffixed(srcfile) + out_suffix + '.out'
     if EMTEST_REBASELINE:
       expected = None
     else:

test/jsrun.py

@@ -11,7 +11,7 @@
 import sys
 from subprocess import PIPE, CalledProcessError
 
-from tools import shared, utils
+from tools import utils
 
 WORKING_ENGINES = {} # Holds all configured engines and whether they work: maps path -> True/False
 DEFAULT_TIMEOUT = 5 * 60
@@ -22,7 +22,7 @@ def make_command(filename, engine, args=None):
   # if no engine is needed, indicated by None, then there is a native executable
   # provided which we can just run
   if engine[0] is None:
-    executable = shared.replace_suffix(os.path.abspath(filename), '.exe')
+    executable = utils.replace_suffix(os.path.abspath(filename), '.exe')
     return [executable] + args
   # Emscripten supports multiple javascript runtimes.  The default is nodejs but
   # it can also use d8 (the v8 engine shell) or jsc (JavaScript Core aka
@@ -43,7 +43,7 @@ def make_command(filename, engine, args=None):
     command_flags += ['run']
   if is_wasmer or is_wasmtime:
     # in a wasm runtime, run the wasm, not the js
-    filename = shared.replace_suffix(filename, '.wasm')
+    filename = utils.replace_suffix(filename, '.wasm')
   # Separates engine flags from script flags
   flag_separator = ['--'] if is_d8 or is_jsc else []
   return engine + command_flags + [filename] + flag_separator + args

test/test_benchmark.py

@@ -24,7 +24,7 @@
 from tools.shared import CLANG_CC, CLANG_CXX
 from common import test_file, read_file, read_binary, needs_make
 from tools.shared import run_process, PIPE, EMCC, config
-from tools import building, utils, shared
+from tools import building, utils
 
 # standard arguments for timing:
 # 0: no runtime, just startup
@@ -250,7 +250,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat
     self.cmd = cmd
     run_process(cmd, env=self.env)
     if self.binaryen_opts:
-      run_binaryen_opts(shared.replace_suffix(final, '.wasm'), self.binaryen_opts)
+      run_binaryen_opts(utils.replace_suffix(final, '.wasm'), self.binaryen_opts)
     self.filename = final
 
   def run(self, args):
@@ -260,12 +260,12 @@ def get_output_files(self):
     ret = [self.filename]
     if 'WASM=0' in self.cmd:
       if 'MINIMAL_RUNTIME=0' not in self.cmd:
-        ret.append(shared.replace_suffix(self.filename, '.asm.js'))
-        ret.append(shared.replace_suffix(self.filename, '.mem'))
+        ret.append(utils.replace_suffix(self.filename, '.asm.js'))
+        ret.append(utils.replace_suffix(self.filename, '.mem'))
       else:
         ret.append(self.filename + '.mem')
     else:
-      ret.append(shared.replace_suffix(self.filename, '.wasm'))
+      ret.append(utils.replace_suffix(self.filename, '.wasm'))
     return ret
 
 
@@ -346,7 +346,7 @@ def run(self, args):
     return jsrun.run_js(self.filename, engine=self.engine, args=args, stderr=PIPE)
 
   def get_output_files(self):
-    return [self.filename, shared.replace_suffix(self.filename, '.wasm')]
+    return [self.filename, utils.replace_suffix(self.filename, '.wasm')]
 
 
 # Benchmarkers

test/test_core.py

@@ -4067,7 +4067,7 @@ def dylink_testf(self, main, side=None, expected=None, force_c=False, main_cflag
     # Same as dylink_test but takes source code as filenames on disc.
     old_args = self.cflags.copy()
     if not expected:
-      outfile = shared.replace_suffix(main, '.out')
+      outfile = utils.replace_suffix(main, '.out')
       expected = read_file(outfile)
     if not side:
       side, ext = os.path.splitext(main)

test/test_other.py

@@ -51,13 +51,13 @@
 from tools.settings import settings
 from tools.system_libs import DETERMINISTIC_PREFIX
 
-emmake = shared.bat_suffix(path_from_root('emmake'))
-emconfig = shared.bat_suffix(path_from_root('em-config'))
-emsize = shared.bat_suffix(path_from_root('emsize'))
-empath_split = shared.bat_suffix(path_from_root('empath-split'))
-emprofile = shared.bat_suffix(path_from_root('emprofile'))
-emstrip = shared.bat_suffix(path_from_root('emstrip'))
-emsymbolizer = shared.bat_suffix(path_from_root('emsymbolizer'))
+emmake = utils.bat_suffix(path_from_root('emmake'))
+emconfig = utils.bat_suffix(path_from_root('em-config'))
+emsize = utils.bat_suffix(path_from_root('emsize'))
+empath_split = utils.bat_suffix(path_from_root('empath-split'))
+emprofile = utils.bat_suffix(path_from_root('emprofile'))
+emstrip = utils.bat_suffix(path_from_root('emstrip'))
+emsymbolizer = utils.bat_suffix(path_from_root('emsymbolizer'))
 
 
 def is_bitcode(filename):
@@ -756,7 +756,7 @@ def test_print_prog_name(self):
     output = self.run_process([EMCC, '--print-prog-name=clang'], stdout=PIPE).stdout
     expected = CLANG_CC
     if WINDOWS:
-      expected = os.path.normpath(shared.unsuffixed(CLANG_CC))
+      expected = os.path.normpath(utils.unsuffixed(CLANG_CC))
     self.assertContained(expected, output)
 
   @crossplatform
@@ -2947,7 +2947,7 @@ def test_js_optimizer(self, passes, filename=None):
       testname = self.id().split('.')[-1]
       filename = utils.removeprefix(testname, 'test_js_optimizer_') + '.js'
     filename = test_file('js_optimizer', filename)
-    expected_file = shared.unsuffixed(filename) + '-output.js'
+    expected_file = utils.unsuffixed(filename) + '-output.js'
     # test calling optimizer
     js = self.run_process(config.NODE_JS + [path_from_root('tools/acorn-optimizer.mjs'), filename] + passes, stdin=PIPE, stdout=PIPE).stdout
     if common.EMTEST_REBASELINE:
@@ -3982,7 +3982,7 @@ def test_file_packager_depfile(self, embed):
     else:
       self.run_process([FILE_PACKAGER, 'test.data', '--js-output=test.js', '--depfile=test.data.d', '--from-emcc', '--preload', '.'])
       output = read_file('test.data.d')
-    file_packager = utils.normalize_path(shared.replace_suffix(FILE_PACKAGER, '.py'))
+    file_packager = utils.normalize_path(utils.replace_suffix(FILE_PACKAGER, '.py'))
     file_packager = file_packager.replace(' ', '\\ ')
     lines = output.splitlines()
     split = lines.index(': \\')
@@ -6903,7 +6903,7 @@ def test_sdl2_config(self):
       [['--cflags', '--libs'], '-sUSE_SDL=2'],
     ]:
       print(args, expected)
-      out = self.run_process([shared.bat_suffix(cache.get_sysroot_dir('bin/sdl2-config'))] + args,
+      out = self.run_process([utils.bat_suffix(cache.get_sysroot_dir('bin/sdl2-config'))] + args,
                              stdout=PIPE, stderr=PIPE).stdout
       self.assertContained(expected, out)
       print('via emmake')

test/test_sanity.py

@@ -488,7 +488,7 @@ def make_readonly(filename):
       with env_modify({'EM_CACHE': self.in_dir('test_cache')}):
         self.run_process([EMCC, test_file('hello_world.c'), '-c'])
     finally:
-      for_all_files(path_from_root('system/include'), shared.make_writable)
+      for_all_files(path_from_root('system/include'), utils.make_writable)
 
   @parameterized({
     '': [False, False],
@@ -831,7 +831,7 @@ def test_bootstrap_without_em_config(self):
 
     # Remove from PATH every directory that contains clang.exe so that bootstrap.py cannot
     # accidentally succeed by virtue of locating tools in PATH.
-    new_path = [d for d in env['PATH'].split(os.pathsep) if not os.path.isfile(os.path.join(d, shared.exe_suffix('clang')))]
+    new_path = [d for d in env['PATH'].split(os.pathsep) if not os.path.isfile(os.path.join(d, utils.exe_suffix('clang')))]
     env['PATH'] = os.pathsep.join(new_path)
 
     # Running bootstrap.py should not fail

tools/building.py

@@ -31,7 +31,7 @@
 from .shared import path_from_root
 from .shared import asmjs_mangle, DEBUG
 from .shared import LLVM_DWARFDUMP, demangle_c_symbol_name
-from .shared import get_emscripten_temp_dir, exe_suffix, is_c_symbol
+from .shared import get_emscripten_temp_dir, is_c_symbol
 from .utils import WINDOWS
 from .settings import settings
 from .feature_matrix import UNSUPPORTED
@@ -384,9 +384,9 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False, work
     return check_call(cmd, stdout=PIPE).stdout
 
   acorn_optimizer.counter += 1
-  basename = shared.unsuffixed(original_filename)
+  basename = utils.unsuffixed(original_filename)
   if '.jso' in basename:
-    basename = shared.unsuffixed(basename)
+    basename = utils.unsuffixed(basename)
   output_file = basename + '.jso%d.js' % acorn_optimizer.counter
   shared.get_temp_files().note(output_file)
   cmd += ['-o', output_file]
@@ -1181,7 +1181,7 @@ def get_binaryen_feature_flags():
 
 
 def check_binaryen(bindir):
-  opt = os.path.join(bindir, exe_suffix('wasm-opt'))
+  opt = os.path.join(bindir, utils.exe_suffix('wasm-opt'))
   if not os.path.exists(opt):
     exit_with_error('binaryen executable not found (%s). Please check your binaryen installation' % opt)
   try:

tools/empath-split.py

@@ -89,7 +89,7 @@ def parse_args():
   if args.preserve_manifest:
     args.verbose = True
   if not args.wasm_split:
-    args.wasm_split = os.path.join(building.get_binaryen_bin(), shared.exe_suffix('wasm-split'))
+    args.wasm_split = os.path.join(building.get_binaryen_bin(), utils.exe_suffix('wasm-split'))
 
   if '--manifest' in forwarded_args:
     parser.error('manifest file will be generated by this script and should not be given')

tools/file_packager.py

@@ -255,7 +255,7 @@ def generate_object_file(data_files):
   embed_files = [f for f in data_files if f.mode == 'embed']
   assert embed_files
 
-  asm_file = shared.replace_suffix(options.obj_output, '.s')
+  asm_file = utils.replace_suffix(options.obj_output, '.s')
 
   used = set()
   for f in embed_files: