Enable support for running the test harness in Safari. (#25499)

Add support for running the `browser` harness with
`EMTEST_BROWSER=/Applications/Safari.App/Contents/MacOS/Safari` on a
macOS system.

This fixes both the singlethreaded and multithreaded harness runs.

Author: juj

test/common.py

@@ -52,6 +52,11 @@
 # used in CI. To use a custom start command specify the executable and command
 # line flags.
 #
+# Note that when specifying EMTEST_BROWSER to run tests on a Safari browser:
+# the command line must point to the root of the app bundle, and not to the
+# Safari executable inside the bundle. I.e. pass EMTEST_BROWSER=/Applications/Safari.app
+# instead of EMTEST_BROWSER=/Applications/Safari.app/Contents/MacOS/Safari
+#
 # There are two special values that can be used here if running in an actual
 # browser is not desired:
 #  EMTEST_BROWSER=0 : This will disable the actual running of the test and simply
@@ -127,6 +132,21 @@ def configure(data_dir):
     shutil.copy(test_file('firefox_user.js'), os.path.join(data_dir, 'user.js'))
 
 
+class SafariConfig:
+  default_flags = ('', )
+  executable_name = 'Safari'
+  # For the macOS 'open' command, pass
+  #   --new: to make a new Safari app be launched, rather than add a tab to an existing Safari process/window
+  #   --fresh: do not restore old tabs (e.g. if user had old navigated windows open)
+  #   --background: Open the new Safari window behind the current Terminal window, to make following the test run more pleasing (this is for convenience only)
+  #   -a <exe_name>: The path to the executable to open, in this case Safari
+  launch_prefix = ('open', '--new', '--fresh', '--background', '-a')
+
+  @staticmethod
+  def configure(data_dir):
+    """ Safari has no special configuration step."""
+
+
 # Special value for passing to assert_returncode which means we expect that program
 # to fail with non-zero return code, but we don't care about specifically which one.
 NON_ZERO = -1
@@ -205,11 +225,17 @@ def is_firefox():
   return EMTEST_BROWSER and 'firefox' in EMTEST_BROWSER.lower()
 
 
+def is_safari():
+  return EMTEST_BROWSER and 'safari' in EMTEST_BROWSER.lower()
+
+
 def get_browser_config():
   if is_chrome():
     return ChromeConfig()
   elif is_firefox():
     return FirefoxConfig()
+  elif is_safari():
+    return SafariConfig()
   return None
 
 
@@ -2216,7 +2242,12 @@ def end_headers(self):
       self.send_header('Cross-Origin-Opener-Policy', 'same-origin')
       self.send_header('Cross-Origin-Embedder-Policy', 'require-corp')
       self.send_header('Cross-Origin-Resource-Policy', 'cross-origin')
-      self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
+
+      self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate, private, max-age=0')
+      self.send_header('Expires', '0')
+      self.send_header('Pragma', 'no-cache')
+      self.send_header('Vary', '*') # Safari insists on caching if this header is not present in addition to the above
+
       return SimpleHTTPRequestHandler.end_headers(self)
 
     def do_POST(self):  # noqa: DC04
@@ -2303,9 +2334,13 @@ def do_GET(self):
           raise Exception('browser harness error, excessive response to server - test must be fixed! "%s"' % self.path)
         self.send_response(200)
         self.send_header('Content-type', 'text/plain')
-        self.send_header('Cache-Control', 'no-cache, must-revalidate')
+
+        self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate, private, max-age=0')
+        self.send_header('Expires', '0')
+        self.send_header('Pragma', 'no-cache')
+        self.send_header('Vary', '*') # Safari insists on caching if this header is not present in addition to the above
+
         self.send_header('Connection', 'close')
-        self.send_header('Expires', '-1')
         self.end_headers()
         self.wfile.write(b'OK')
 
@@ -2558,11 +2593,17 @@ def browser_restart(cls):
   def browser_open(cls, url):
     assert has_browser()
     browser_args = EMTEST_BROWSER
+    parallel_harness = worker_id is not None
+
+    config = get_browser_config()
+    if not config and EMTEST_BROWSER_AUTO_CONFIG:
+      exit_with_error(f'EMTEST_BROWSER_AUTO_CONFIG only currently works with firefox, chrome and safari. EMTEST_BROWSER was "{EMTEST_BROWSER}"')
 
-    if EMTEST_BROWSER_AUTO_CONFIG:
+    # Prepare the browser data directory, if it uses one.
+    if EMTEST_BROWSER_AUTO_CONFIG and config and hasattr(config, 'data_dir_flag'):
       logger.info('Using default CI configuration.')
       browser_data_dir = DEFAULT_BROWSER_DATA_DIR
-      if worker_id is not None:
+      if parallel_harness:
         # Running in parallel mode, give each browser its own profile dir.
         browser_data_dir += '-' + str(worker_id)
 
@@ -2577,49 +2618,58 @@ def browser_open(cls, url):
       # Recreate the new data directory.
       os.mkdir(browser_data_dir)
 
-      config = get_browser_config()
-      if not config:
-        exit_with_error(f'EMTEST_BROWSER_AUTO_CONFIG only currently works with firefox or chrome. EMTEST_BROWSER was "{EMTEST_BROWSER}"')
       if WINDOWS:
         # Escape directory delimiter backslashes for shlex.split.
         browser_data_dir = browser_data_dir.replace('\\', '\\\\')
       config.configure(browser_data_dir)
       browser_args += f' {config.data_dir_flag}"{browser_data_dir}"'
 
     browser_args = shlex.split(browser_args)
+    if hasattr(config, 'launch_prefix'):
+      browser_args = list(config.launch_prefix) + browser_args
+
     logger.info('Launching browser: %s', str(browser_args))
 
-    if WINDOWS and is_firefox():
-      cls.launch_browser_harness_windows_firefox(worker_id, config, browser_args, url)
+    if (WINDOWS and is_firefox()) or is_safari():
+      cls.launch_browser_harness_with_proc_snapshot_workaround(parallel_harness, config, browser_args, url)
     else:
       cls.browser_procs = [subprocess.Popen(browser_args + [url])]
 
   @classmethod
-  def launch_browser_harness_windows_firefox(cls, worker_id, config, browser_args, url):
-    ''' Dedicated function for launching browser harness on Firefox on Windows,
-    which requires extra care for window positioning and process tracking.'''
+  def launch_browser_harness_with_proc_snapshot_workaround(cls, parallel_harness, config, browser_args, url):
+    ''' Dedicated function for launching browser harness in scenarios where
+    we need to identify the launched browser processes via a before-after
+    subprocess snapshotting delta workaround.'''
 
+    # In order for this to work, each browser needs to be launched one at a time
+    # so that we know which process belongs to which browser.
     with FileLock(browser_spawn_lock_filename) as count:
-      # Firefox is a multiprocess browser. On Windows, killing the spawned
-      # process will not bring down the whole browser, but only one browser tab.
-      # So take a delta snapshot before->after spawning the browser to find
-      # which subprocesses we launched.
-      if worker_id is not None:
+      # Take a snapshot before spawning the browser to find which processes
+      # existed before launching the browser.
+      if parallel_harness or is_safari():
         procs_before = list_processes_by_name(config.executable_name)
+
+      # Browser launch
       cls.browser_procs = [subprocess.Popen(browser_args + [url])]
-      # Give Firefox time to spawn its subprocesses. Use an increasing timeout
-      # as a crude way to account for system load.
-      if worker_id is not None:
+
+      # Give the browser time to spawn its subprocesses. Use an increasing
+      # timeout as a crude way to account for system load.
+      if parallel_harness or is_safari():
         time.sleep(2 + count * 0.3)
         procs_after = list_processes_by_name(config.executable_name)
+
+        # Take a snapshot again to find which processes exist after launching
+        # the browser. Then the newly launched browser processes are determined
+        # by the delta before->after.
+        cls.browser_procs = list(set(procs_after).difference(set(procs_before)))
+        if len(cls.browser_procs) == 0:
+          exit_with_error('Could not detect the launched browser subprocesses. The test harness will not be able to close the browser after testing is done, so aborting the test run here.')
+
+      # Firefox on Windows quirk:
       # Make sure that each browser window is visible on the desktop. Otherwise
       # browser might decide that the tab is backgrounded, and not load a test,
       # or it might not tick rAF()s forward, causing tests to hang.
-      if worker_id is not None and not EMTEST_HEADLESS:
-        # On Firefox on Windows we needs to track subprocesses that got created
-        # by Firefox. Other setups can use 'browser_proc' directly to terminate
-        # the browser.
-        cls.browser_procs = list(set(procs_after).difference(set(procs_before)))
+      if WINDOWS and parallel_harness and not EMTEST_HEADLESS:
         # Wrap window positions on a Full HD desktop area modulo primes.
         for proc in cls.browser_procs:
           move_browser_window(proc.pid, (300 + count * 47) % 1901, (10 + count * 37) % 997)
@@ -2693,6 +2743,7 @@ def run_browser(self, html_file, expected=None, message=None, timeout=None, extr
     if DEBUG:
       print('[browser launch:', html_file, ']')
     assert not (message and expected), 'run_browser expects `expected` or `message`, but not both'
+
     if expected is not None:
       try:
         self.harness_in_queue.put((

test/parallel_testsuite.py

@@ -36,7 +36,7 @@ def python_multiprocessing_structures_are_buggy():
 def cap_max_workers_in_pool(max_workers, is_browser):
   if is_browser and 'EMTEST_CORES' not in os.environ and 'EMCC_CORES' not in os.environ:
     # TODO experiment with this number. In browser tests we'll be creating
-    # a chrome instance per worker which is expensive.
+    # a browser instance per worker which is expensive.
     max_workers = max_workers // 2
   # Python has an issue that it can only use max 61 cores on Windows: https://github.com/python/cpython/issues/89240
   if WINDOWS:

test/test_browser.py

@@ -18,7 +18,7 @@
 from urllib.request import urlopen
 
 import common
-from common import BrowserCore, RunnerCore, path_from_root, has_browser, Reporting, is_chrome, is_firefox, CHROMIUM_BASED_BROWSERS
+from common import BrowserCore, RunnerCore, path_from_root, has_browser, Reporting, is_chrome, is_firefox, is_safari, CHROMIUM_BASED_BROWSERS
 from common import create_file, parameterized, ensure_dir, disabled, flaky, test_file, WEBIDL_BINDER
 from common import read_file, EMRUN, no_wasm64, no_2gb, no_4gb, copytree, skip_if, skip_if_simple
 from common import requires_wasm2js, parameterize, find_browser_test_file, with_all_sjlj
@@ -40,7 +40,12 @@ def sendheaders(s, extra=None, length=None):
       s.send_header("Content-Length", str(length))
       s.send_header("Access-Control-Allow-Origin", "http://localhost:%s" % port)
       s.send_header('Cross-Origin-Resource-Policy', 'cross-origin')
-      s.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
+
+      s.send_header('Cache-Control', 'no-cache, no-store, must-revalidate, private, max-age=0')
+      s.send_header('Expires', '0')
+      s.send_header('Pragma', 'no-cache')
+      s.send_header('Vary', '*') # Safari insists on caching if this header is not present in addition to the above
+
       s.send_header("Access-Control-Expose-Headers", "Content-Length, Accept-Ranges")
       s.send_header("Content-type", "application/octet-stream")
       if support_byte_ranges:
@@ -142,6 +147,8 @@ def is_swiftshader(_):
 
 no_firefox = skip_if('no_firefox', lambda _: is_firefox(), 'firefox is not supported')
 
+no_safari = skip_if('no_safari', lambda _: is_safari(), 'safari is not supported')
+
 
 def is_jspi(args):
   return '-sASYNCIFY=2' in args
@@ -2797,6 +2804,7 @@ def test_webgl2_pbo(self):
     self.btest_exit('webgl2_pbo.c', cflags=['-sMAX_WEBGL_VERSION=2', '-lGL'])
 
   @no_firefox('fails on CI likely due to GPU drivers there')
+  @no_safari('TODO: Fails with report_result?5') # Safari 17.6 (17618.3.11.11.7, 17618)
   @requires_graphics_hardware
   def test_webgl2_sokol_mipmap(self):
     self.reftest('third_party/sokol/mipmap-emsc.c', 'third_party/sokol/mipmap-emsc.png',
@@ -2955,6 +2963,7 @@ def test_sdl2_image_jpeg(self):
   @also_with_wasmfs
   @requires_graphics_hardware
   @with_all_sjlj
+  @no_safari('Test enables Wasm exceptions, so will not run in Safari 17.6 (17618.3.11.11.7, 17618)') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_sdl2_image_formats(self):
     shutil.copy(test_file('screenshot.png'), '.')
     shutil.copy(test_file('screenshot.jpg'), '.')
@@ -3356,6 +3365,7 @@ def test_async_returnvalue(self, args):
       create_file('filey.txt', 'sync_tunnel\nsync_tunnel_bool\n')
     self.btest('test_async_returnvalue.c', '0', cflags=['-sASSERTIONS', '-sASYNCIFY', '-sASYNCIFY_IGNORE_INDIRECT', '--js-library', test_file('browser/test_async_returnvalue.js')] + args)
 
+  @no_safari('TODO: Never reports a result, so times out') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_async_bad_list(self):
     self.btest('test_async_bad_list.c', '0', cflags=['-sASYNCIFY', '-sASYNCIFY_ONLY=waka', '--profiling'])
 
@@ -3408,6 +3418,7 @@ def test_modularize(self, args, code, opts):
     self.run_browser('a.html', '/report_result?0')
 
   @no_firefox('source phase imports not implemented yet in firefox')
+  @no_safari('croaks on line "import source wasmModule from \'./out.wasm\';"') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_source_phase_imports(self):
     self.compile_btest('browser_test_hello_world.c', ['-sEXPORT_ES6', '-sSOURCE_PHASE_IMPORTS', '-Wno-experimental', '-o', 'out.mjs'])
     create_file('a.html', '''
@@ -3794,6 +3805,7 @@ def test_pthread_gcc_64bit_atomic_fetch_and_op(self):
 
   # Test the old GCC atomic __sync_op_and_fetch builtin operations.
   @also_with_wasm2js
+  @no_safari('TODO: browser.test_pthread_gcc_atomic_op_and_fetch_wasm2js fails with "abort:Assertion failed: nand_and_fetch_data == -1"') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_pthread_gcc_atomic_op_and_fetch(self):
     self.cflags += ['-Wno-sync-fetch-and-nand-semantics-changed']
     self.btest_exit('pthread/test_pthread_gcc_atomic_op_and_fetch.c', cflags=['-O3', '-pthread', '-sPTHREAD_POOL_SIZE=8'])
@@ -3891,6 +3903,7 @@ def test_pthread_cleanup(self):
     '': ([],),
     'spinlock': (['-DSPINLOCK_TEST'],),
   })
+  @no_safari('TODO: browser.test_pthread_mutex and browser.test_pthread_mutex_spinlock both hang Safari') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_pthread_mutex(self, args):
     self.btest_exit('pthread/test_pthread_mutex.c', cflags=['-O3', '-pthread', '-sPTHREAD_POOL_SIZE=8'] + args)
 
@@ -4078,6 +4091,7 @@ def test_pthread_safe_stack(self):
     'no_leak': ['test_pthread_lsan_no_leak', []],
   })
   @no_firefox('https://github.com/emscripten-core/emscripten/issues/15978')
+  @no_safari('TODO: browser.test_pthread_lsan_leak fails with /report_result?0') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_pthread_lsan(self, name, args):
     self.btest(Path('pthread', name + '.cpp'), expected='1', cflags=['-fsanitize=leak', '-pthread', '-sPROXY_TO_PTHREAD', '--pre-js', test_file('pthread', name + '.js')] + args)
 
@@ -4089,6 +4103,7 @@ def test_pthread_lsan(self, name, args):
     'leak': ['test_pthread_lsan_leak', ['-gsource-map']],
     'no_leak': ['test_pthread_lsan_no_leak', []],
   })
+  @no_safari('TODO: browser.test_pthread_asan_leak fails with /report_result?0') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_pthread_asan(self, name, args):
     self.btest(Path('pthread', name + '.cpp'), expected='1', cflags=['-fsanitize=address', '-pthread', '-sPROXY_TO_PTHREAD', '--pre-js', test_file('pthread', name + '.js')] + args)
 
@@ -4102,6 +4117,7 @@ def test_pthread_asan_use_after_free(self):
   @no_2gb('ASAN + GLOBAL_BASE')
   @no_4gb('ASAN + GLOBAL_BASE')
   @no_firefox('https://github.com/emscripten-core/emscripten/issues/20006')
+  @no_safari('TODO: Hangs') # Safari Version 18.5 (20621.2.5.11.8)
   @also_with_wasmfs
   def test_pthread_asan_use_after_free_2(self):
     # similiar to test_pthread_asan_use_after_free, but using a pool instead
@@ -4120,6 +4136,7 @@ def test_pthread_exit_process(self):
     args += ['--pre-js', test_file('core/pthread/test_pthread_exit_runtime.pre.js')]
     self.btest('core/pthread/test_pthread_exit_runtime.c', expected='onExit status: 42', cflags=args)
 
+  @no_safari('TODO: Fails with report_result?unexpected: [object ErrorEvent]') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_pthread_trap(self):
     create_file('pre.js', '''
     if (typeof window === 'object' && window) {
@@ -5376,6 +5393,7 @@ def test_wasmfs_fetch_backend_threaded(self, args):
     'jspi': (['-Wno-experimental', '-sASYNCIFY=2'],),
     'jspi_wasm_bigint': (['-Wno-experimental', '-sASYNCIFY=2', '-sWASM_BIGINT'],),
   })
+  @no_safari('TODO: Fails with abort:Assertion failed: err == 0') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_wasmfs_opfs(self, args):
     if '-sASYNCIFY=2' in args:
       self.require_jspi()
@@ -5385,6 +5403,7 @@ def test_wasmfs_opfs(self, args):
     self.btest_exit(test, cflags=args + ['-DWASMFS_RESUME'])
 
   @no_firefox('no OPFS support yet')
+  @no_safari('TODO: Fails with exception:Did not get expected EIO when unlinking file') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_wasmfs_opfs_errors(self):
     test = test_file('wasmfs/wasmfs_opfs_errors.c')
     postjs = test_file('wasmfs/wasmfs_opfs_errors_post.js')
@@ -5459,6 +5478,7 @@ def test_manual_pthread_proxy_hammer(self, args):
   def test_assert_failure(self):
     self.btest('test_assert_failure.c', 'abort:Assertion failed: false && "this is a test"')
 
+  @no_safari('TODO: Fails with report_result?exception:rejected! / undefined') # Safari 17.6 (17618.3.11.11.7, 17618)
   def test_pthread_unhandledrejection(self):
     # Check that an unhandled promise rejection is propagated to the main thread
     # as an error. This test is failing if it hangs!