Safari test skips (#25528)

Improve Safari test harness skips to be aware of Safari version.

Author: juj

test/test_browser.py

@@ -5,7 +5,9 @@
 
 import argparse
 import os
+import plistlib
 import random
+import re
 import shlex
 import shutil
 import subprocess
@@ -26,8 +28,9 @@
 from common import HttpServerThread, requires_dev_dependency
 from tools import shared
 from tools import ports
+from tools.feature_matrix import UNSUPPORTED
 from tools.shared import EMCC, WINDOWS, FILE_PACKAGER, PIPE, DEBUG
-from tools.utils import delete_dir
+from tools.utils import delete_dir, memoize
 
 
 def make_test_chunked_synchronous_xhr_server(support_byte_ranges, data, port):
@@ -141,6 +144,20 @@ def is_swiftshader(_):
   return is_chrome() and '--use-gl=swiftshader' in common.EMTEST_BROWSER
 
 
+@memoize
+def get_safari_version():
+  if not is_safari():
+    return UNSUPPORTED
+  plist_path = os.path.join(common.EMTEST_BROWSER.strip(), 'Contents', 'version.plist')
+  version_str = plistlib.load(open(plist_path, 'rb')).get('CFBundleShortVersionString')
+  # Split into parts (major.minor.patch)
+  parts = (version_str.split('.') + ['0', '0', '0'])[:3]
+  # Convert each part into integers, discarding any trailing string, e.g. '13a' -> 13.
+  parts = [int(re.match(r"\d+", s).group()) if re.match(r"\d+", s) else 0 for s in parts]
+  # Return version as XXYYZZ
+  return parts[0] * 10000 + parts[1] * 100 + parts[2]
+
+
 no_swiftshader = skip_if_simple('not compatible with swiftshader', is_swiftshader)
 
 no_chrome = skip_if('no_chrome', lambda _: is_chrome(), 'chrome is not supported')
@@ -150,6 +167,18 @@ def is_swiftshader(_):
 no_safari = skip_if('no_safari', lambda _: is_safari(), 'safari is not supported')
 
 
+def requires_version(name, version_getter):
+  assert callable(version_getter)
+
+  def decorator(min_required_version, note=''):
+    return skip_if_simple(name, lambda _: version_getter() < min_required_version, f'{name} v{version_getter()} is not supported (need v{min_required_version} at minimum) {note}')
+
+  return decorator
+
+
+requires_safari_version = requires_version('safari', get_safari_version)
+
+
 def is_jspi(args):
   return '-sASYNCIFY=2' in args
 
@@ -289,6 +318,7 @@ def test_sdl1(self):
   def test_sdl1_es6(self):
     self.reftest('hello_world_sdl.c', 'htmltest.png', cflags=['-sUSE_SDL', '-lGL', '-sEXPORT_ES6'])
 
+  @no_safari('TODO: Fails in NEW Safari 26.0.1 (21622.1.22.11.15), but not in OLD Safari 17.6 (17618.3.11.11.7, 17618) or Safari 18.5 (20621.2.5.11.8)')
   def test_emscripten_log(self):
     self.btest_exit('test_emscripten_log.cpp', cflags=['-Wno-deprecated-pragma', '-gsource-map', '-g2'])
 
@@ -1443,6 +1473,7 @@ def test_separate_metadata_later(self):
     self.run_process([FILE_PACKAGER, 'more.data', '--preload', 'data.dat', '--separate-metadata', '--js-output=more.js'])
     self.btest(Path('browser/separate_metadata_later.cpp'), '1', cflags=['-sFORCE_FILESYSTEM'])
 
+  @requires_safari_version(260001, 'TODO: Fails with "Assertion failed: false"') # Fails in Safari 18.5 (20621.2.5.11.8) with Intel x64 CPU only. Passes on Safari 18.5 (20621.2.5.11.8) with ARM M1, Safari 17.6 (17618.3.11.11.7, 17618)/x64 and Safari 26.0.1 (21622.1.22.11.15)/M4
   def test_idbstore(self):
     secret = str(time.time())
     for stage in (0, 1, 2, 3, 0, 1, 2, 0, 0, 1, 4, 2, 5, 0, 4, 6, 5):
@@ -2800,7 +2831,7 @@ def test_webgl2_pbo(self):
     self.btest_exit('webgl2_pbo.c', cflags=['-sMAX_WEBGL_VERSION=2', '-lGL'])
 
   @no_firefox('fails on CI likely due to GPU drivers there')
-  @no_safari('TODO: Fails with report_result?5') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @no_safari('TODO: Fails with report_result?5') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), Safari 26.0.1 (21622.1.22.11.15)
   @requires_graphics_hardware
   def test_webgl2_sokol_mipmap(self):
     self.reftest('third_party/sokol/mipmap-emsc.c', 'third_party/sokol/mipmap-emsc.png',
@@ -2959,7 +2990,7 @@ def test_sdl2_image_jpeg(self):
   @also_with_wasmfs
   @requires_graphics_hardware
   @with_all_sjlj
-  @no_safari('Test enables Wasm exceptions, so will not run in Safari 17.6 (17618.3.11.11.7, 17618)') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @requires_safari_version(170601, 'TODO: Test enables Wasm exceptions') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), passes in Safari 18.5 (20621.2.5.11.8) and Safari 26.0.1 (21622.1.22.11.15)
   def test_sdl2_image_formats(self):
     shutil.copy(test_file('screenshot.png'), '.')
     shutil.copy(test_file('screenshot.jpg'), '.')
@@ -3361,7 +3392,7 @@ def test_async_returnvalue(self, args):
       create_file('filey.txt', 'sync_tunnel\nsync_tunnel_bool\n')
     self.btest('test_async_returnvalue.c', '0', cflags=['-sASSERTIONS', '-sASYNCIFY', '-sASYNCIFY_IGNORE_INDIRECT', '--js-library', test_file('browser/test_async_returnvalue.js')] + args)
 
-  @no_safari('TODO: Never reports a result, so times out') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @no_safari('TODO: Never reports a result, so times out') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), Safari 26.0.1 (21622.1.22.11.15)
   def test_async_bad_list(self):
     self.btest('test_async_bad_list.c', '0', cflags=['-sASYNCIFY', '-sASYNCIFY_ONLY=waka', '--profiling'])
 
@@ -3414,7 +3445,7 @@ def test_modularize(self, args, code, opts):
     self.run_browser('a.html', '/report_result?0')
 
   @no_firefox('source phase imports not implemented yet in firefox')
-  @no_safari('croaks on line "import source wasmModule from \'./out.wasm\';"') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @no_safari('TODO: croaks on line "import source wasmModule from \'./out.wasm\';"') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), Safari 26.0.1 (21622.1.22.11.15)
   def test_source_phase_imports(self):
     self.compile_btest('browser_test_hello_world.c', ['-sEXPORT_ES6', '-sSOURCE_PHASE_IMPORTS', '-Wno-experimental', '-o', 'out.mjs'])
     create_file('a.html', '''
@@ -3801,7 +3832,7 @@ def test_pthread_gcc_64bit_atomic_fetch_and_op(self):
 
   # Test the old GCC atomic __sync_op_and_fetch builtin operations.
   @also_with_wasm2js
-  @no_safari('TODO: browser.test_pthread_gcc_atomic_op_and_fetch_wasm2js fails with "abort:Assertion failed: nand_and_fetch_data == -1"') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @requires_safari_version(170601, 'TODO: browser.test_pthread_gcc_atomic_op_and_fetch_wasm2js fails with "abort:Assertion failed: nand_and_fetch_data == -1"') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), passes in Safari 18.5 (20621.2.5.11.8) and Safari 26.0.1 (21622.1.22.11.15)
   def test_pthread_gcc_atomic_op_and_fetch(self):
     self.cflags += ['-Wno-sync-fetch-and-nand-semantics-changed']
     self.btest_exit('pthread/test_pthread_gcc_atomic_op_and_fetch.c', cflags=['-O3', '-pthread', '-sPTHREAD_POOL_SIZE=8'])
@@ -3899,7 +3930,7 @@ def test_pthread_cleanup(self):
     '': ([],),
     'spinlock': (['-DSPINLOCK_TEST'],),
   })
-  @no_safari('TODO: browser.test_pthread_mutex and browser.test_pthread_mutex_spinlock both hang Safari') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @requires_safari_version(170601, 'TODO: browser.test_pthread_mutex and browser.test_pthread_mutex_spinlock both hang Safari') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), passes in Safari 18.5 (20621.2.5.11.8) and Safari 26.0.1 (21622.1.22.11.15)
   def test_pthread_mutex(self, args):
     self.btest_exit('pthread/test_pthread_mutex.c', cflags=['-O3', '-pthread', '-sPTHREAD_POOL_SIZE=8'] + args)
 
@@ -4087,7 +4118,7 @@ def test_pthread_safe_stack(self):
     'no_leak': ['test_pthread_lsan_no_leak', []],
   })
   @no_firefox('https://github.com/emscripten-core/emscripten/issues/15978')
-  @no_safari('TODO: browser.test_pthread_lsan_leak fails with /report_result?0') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @no_safari('TODO: browser.test_pthread_lsan_leak fails with /report_result?0') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), Safari 26.0.1 (21622.1.22.11.15)
   def test_pthread_lsan(self, name, args):
     self.btest(Path('pthread', name + '.cpp'), expected='1', cflags=['-fsanitize=leak', '-pthread', '-sPROXY_TO_PTHREAD', '--pre-js', test_file('pthread', name + '.js')] + args)
 
@@ -4099,7 +4130,7 @@ def test_pthread_lsan(self, name, args):
     'leak': ['test_pthread_lsan_leak', ['-gsource-map']],
     'no_leak': ['test_pthread_lsan_no_leak', []],
   })
-  @no_safari('TODO: browser.test_pthread_asan_leak fails with /report_result?0') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @no_safari('TODO: browser.test_pthread_asan_leak fails with /report_result?0') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), Safari 26.0.1 (21622.1.22.11.15)
   def test_pthread_asan(self, name, args):
     self.btest(Path('pthread', name + '.cpp'), expected='1', cflags=['-fsanitize=address', '-pthread', '-sPROXY_TO_PTHREAD', '--pre-js', test_file('pthread', name + '.js')] + args)
 
@@ -4113,7 +4144,7 @@ def test_pthread_asan_use_after_free(self):
   @no_2gb('ASAN + GLOBAL_BASE')
   @no_4gb('ASAN + GLOBAL_BASE')
   @no_firefox('https://github.com/emscripten-core/emscripten/issues/20006')
-  @no_safari('TODO: Hangs') # Safari Version 18.5 (20621.2.5.11.8)
+  @no_safari('TODO: Hangs') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), Safari 26.0.1 (21622.1.22.11.15)
   @also_with_wasmfs
   def test_pthread_asan_use_after_free_2(self):
     # similiar to test_pthread_asan_use_after_free, but using a pool instead
@@ -4132,7 +4163,7 @@ def test_pthread_exit_process(self):
     args += ['--pre-js', test_file('core/pthread/test_pthread_exit_runtime.pre.js')]
     self.btest('core/pthread/test_pthread_exit_runtime.c', expected='onExit status: 42', cflags=args)
 
-  @no_safari('TODO: Fails with report_result?unexpected: [object ErrorEvent]') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @no_safari('TODO: Fails with report_result?unexpected: [object ErrorEvent]') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), Safari 26.0.1 (21622.1.22.11.15)
   def test_pthread_trap(self):
     create_file('pre.js', '''
     if (typeof window === 'object' && window) {
@@ -5321,6 +5352,7 @@ def test_4gb(self):
 
   # Tests that emmalloc supports up to 4GB Wasm heaps.
   @no_firefox('no 4GB support yet')
+  @requires_safari_version(170601, 'Assertion failed: emscripten_get_heap_size() == MAX_HEAP') # Fails in Safari 17.6 (17618.3.11.11.7, 17618)
   @no_4gb('uses MAXIMUM_MEMORY')
   def test_emmalloc_4gb(self):
     # For now, keep this in browser as this suite runs serially, which
@@ -5389,7 +5421,7 @@ def test_wasmfs_fetch_backend_threaded(self, args):
     'jspi': (['-Wno-experimental', '-sASYNCIFY=2'],),
     'jspi_wasm_bigint': (['-Wno-experimental', '-sASYNCIFY=2', '-sWASM_BIGINT'],),
   })
-  @no_safari('TODO: Fails with abort:Assertion failed: err == 0') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @no_safari('TODO: Fails with abort:Assertion failed: err == 0') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), Safari 26.0.1 (21622.1.22.11.15)
   def test_wasmfs_opfs(self, args):
     if '-sASYNCIFY=2' in args:
       self.require_jspi()
@@ -5399,7 +5431,7 @@ def test_wasmfs_opfs(self, args):
     self.btest_exit(test, cflags=args + ['-DWASMFS_RESUME'])
 
   @no_firefox('no OPFS support yet')
-  @no_safari('TODO: Fails with exception:Did not get expected EIO when unlinking file') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @no_safari('TODO: Fails with exception:Did not get expected EIO when unlinking file') # Fails in Safari 17.6 (17618.3.11.11.7, 17618) and Safari 26.0.1 (21622.1.22.11.15)
   def test_wasmfs_opfs_errors(self):
     test = test_file('wasmfs/wasmfs_opfs_errors.c')
     postjs = test_file('wasmfs/wasmfs_opfs_errors_post.js')
@@ -5474,7 +5506,7 @@ def test_manual_pthread_proxy_hammer(self, args):
   def test_assert_failure(self):
     self.btest('test_assert_failure.c', 'abort:Assertion failed: false && "this is a test"')
 
-  @no_safari('TODO: Fails with report_result?exception:rejected! / undefined') # Safari 17.6 (17618.3.11.11.7, 17618)
+  @no_safari('TODO: Fails with report_result?exception:rejected!') # Fails in Safari 17.6 (17618.3.11.11.7, 17618), Safari 26.0.1 (21622.1.22.11.15)
   def test_pthread_unhandledrejection(self):
     # Check that an unhandled promise rejection is propagated to the main thread
     # as an error. This test is failing if it hangs!