From 42247b2bcf78006438b44035934e61d0526f559e Mon Sep 17 00:00:00 2001 From: slipher Date: Tue, 2 Sep 2025 07:18:20 -0500 Subject: [PATCH 1/8] Crash test script WIP --- tools/crash_test.py | 113 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 tools/crash_test.py diff --git a/tools/crash_test.py b/tools/crash_test.py new file mode 100644 index 0000000000..9f84b25af5 --- /dev/null +++ b/tools/crash_test.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +import traceback + +DAEMON_DIR = '/unv/Unvanquished/daemon' +BREAKPAD_DIR = '/unv/Unvanquished/daemon/libs/breakpad' +GAME_BUILD_DIR = '.' +ARCH = 'amd64' + +SYMBOLIZE = os.path.join(BREAKPAD_DIR, "symbolize.py") +STACKWALK = os.path.join(BREAKPAD_DIR, "src/processor/minidump_stackwalk") + +DAEMON_TTYCLIENT = os.path.join(GAME_BUILD_DIR, 'daemon-tty') +DAEMON_SERVER = os.path.join(GAME_BUILD_DIR, 'daemonded') +TEMP_DIR = os.path.join(GAME_BUILD_DIR, "crashtest-tmp") +os.makedirs(TEMP_DIR, exist_ok=True) +SYMBOL_DIR = os.path.join(TEMP_DIR, "symbols") +os.makedirs(SYMBOL_DIR, exist_ok=True) +HOMEPATH = os.path.join(TEMP_DIR, "homepath") + +dummy = False +if dummy: + GAMELOGIC_NACL = os.path.join(GAME_BUILD_DIR, f"cgame-{ARCH}.nexe") +else: + GAMELOGIC_NACL = os.path.join(GAME_BUILD_DIR, f"sgame-{ARCH}.nexe") + + +assert os.path.isfile(GAMELOGIC_NACL), GAMELOGIC_NACL +assert os.path.isfile(DAEMON_SERVER) +assert os.path.isfile(STACKWALK) + +print(f"Symbolizing '{GAMELOGIC_NACL}'...") +subprocess.check_call([sys.executable, SYMBOLIZE, "--symbol-directory", SYMBOL_DIR, GAMELOGIC_NACL]) + +class CrashTest: + def __init__(self, name): + self.name = name + + def Begin(self): + self.status = "PASSED" + print(f"===RUNNING: {self.name}===") + self.tmpdir = os.path.join(TEMP_DIR, self.name) + os.makedirs(self.tmpdir, exist_ok=True) + + def End(self): + print(f"==={self.status}: {self.name}===") + + def Verify(self, cond, reason): + if not cond: + print(f"FAILURE: {reason}") + self.status = "FAILED" + + def Go(self): + self.Begin() + try: + self.Do() + except Exception as e: + traceback.print_exception(e) + self.status = "FAILED" + self.End() + return self.status == "PASSED" + +class NaclCrashTest(CrashTest): + def __init__(self, fault): + super().__init__(f"nacl.{fault}") + self.fault = fault + + def Do(self): + print("Running daemon...") + p = subprocess.run([DAEMON_SERVER, "-set", "vm.sgame.type", "1", + #"-set", "logs.level.fs", "warning", + "-set", "sv_fps", "1000", + #"-set", "server.private", "2", + #"-set", "sv_networkScope", "0", + "-set", "net_enabled", "0", + "-set", "common.framerate.max", "0", + #"-homepath", HOMEPATH, + #"-pakpath", os.path.join(DAEMON_DIR, "pkg"), + #"-set", "fs_basepak", "testdata", + "+devmap plat23", + "+delay 20f echo CRASHTEST_BEGIN", + "+delay 20f sgame.injectFault", self.fault, + "+delay 20f echo CRASHTEST_END", + "+delay 40f quit"], + stderr=subprocess.PIPE) + log = [s.strip() for s in p.stderr.decode("utf8").splitlines()] + i = log.index("CRASHTEST_BEGIN") + j = log.index("CRASHTEST_END") + # TODO expected vs. actual Warn's + DUMP_PREFIX = "Wrote crash dump to " + dumps = [l for l in log if l.startswith(DUMP_PREFIX)] + assert len(dumps) == 1, "Daemon log contains 1 crash dump" + dump = dumps[0][len(DUMP_PREFIX):] + sw_out = os.path.join(self.tmpdir, "stackwalk.log") + with open(sw_out, "a+") as sw_f: + print(f"Extracting stack trace to '{sw_out}'...") + sw_f.truncate() + subprocess.run([STACKWALK, dump, SYMBOL_DIR], check=True, stdout=sw_f, stderr=subprocess.STDOUT) + sw_f.seek(0) + sw = sw_f.read() + TRACE_FUNC = "InjectFaultCmd::Run" + self.Verify(TRACE_FUNC in sw, "function names not found in trace (did you build with symbols?)") + +passed = (True + & NaclCrashTest("exception").Go() + & NaclCrashTest("throw").Go() + & NaclCrashTest("abort").Go() + & NaclCrashTest("segfault").Go()) +sys.exit(1 - passed) + From 369277c40fef2096314d47ceed2e2d05a774ffed Mon Sep 17 00:00:00 2001 From: slipher Date: Tue, 2 Sep 2025 07:18:20 -0500 Subject: [PATCH 2/8] blah --- tools/crash_test.py | 61 +++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/tools/crash_test.py b/tools/crash_test.py index 9f84b25af5..225834982a 100644 --- a/tools/crash_test.py +++ b/tools/crash_test.py @@ -21,19 +21,11 @@ os.makedirs(SYMBOL_DIR, exist_ok=True) HOMEPATH = os.path.join(TEMP_DIR, "homepath") -dummy = False -if dummy: - GAMELOGIC_NACL = os.path.join(GAME_BUILD_DIR, f"cgame-{ARCH}.nexe") -else: - GAMELOGIC_NACL = os.path.join(GAME_BUILD_DIR, f"sgame-{ARCH}.nexe") -assert os.path.isfile(GAMELOGIC_NACL), GAMELOGIC_NACL assert os.path.isfile(DAEMON_SERVER) assert os.path.isfile(STACKWALK) -print(f"Symbolizing '{GAMELOGIC_NACL}'...") -subprocess.check_call([sys.executable, SYMBOLIZE, "--symbol-directory", SYMBOL_DIR, GAMELOGIC_NACL]) class CrashTest: def __init__(self, name): @@ -42,8 +34,6 @@ def __init__(self, name): def Begin(self): self.status = "PASSED" print(f"===RUNNING: {self.name}===") - self.tmpdir = os.path.join(TEMP_DIR, self.name) - os.makedirs(self.tmpdir, exist_ok=True) def End(self): print(f"==={self.status}: {self.name}===") @@ -64,37 +54,43 @@ def Go(self): return self.status == "PASSED" class NaclCrashTest(CrashTest): - def __init__(self, fault): + def __init__(self, engine, tprefix, fault): super().__init__(f"nacl.{fault}") + self.engine = engine + self.tprefix = tprefix self.fault = fault def Do(self): print("Running daemon...") - p = subprocess.run([DAEMON_SERVER, "-set", "vm.sgame.type", "1", + p = subprocess.run([self.engine, "-set", "vm.sgame.type", "1", + "-set", "vm.cgame.type", "1", #"-set", "logs.level.fs", "warning", "-set", "sv_fps", "1000", + "-set", "common.framerate.max", "0", #"-set", "server.private", "2", #"-set", "sv_networkScope", "0", "-set", "net_enabled", "0", "-set", "common.framerate.max", "0", + "-pakpath", "/unv/Unvanquished/pkg", #"-homepath", HOMEPATH, #"-pakpath", os.path.join(DAEMON_DIR, "pkg"), #"-set", "fs_basepak", "testdata", "+devmap plat23", "+delay 20f echo CRASHTEST_BEGIN", - "+delay 20f sgame.injectFault", self.fault, + f"+delay 20f {self.tprefix}injectFault", self.fault, "+delay 20f echo CRASHTEST_END", "+delay 40f quit"], - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, check=False) log = [s.strip() for s in p.stderr.decode("utf8").splitlines()] i = log.index("CRASHTEST_BEGIN") - j = log.index("CRASHTEST_END") + #j = log.index("CRASHTEST_END") + j = len(log) - 1 # TODO expected vs. actual Warn's DUMP_PREFIX = "Wrote crash dump to " dumps = [l for l in log if l.startswith(DUMP_PREFIX)] assert len(dumps) == 1, "Daemon log contains 1 crash dump" dump = dumps[0][len(DUMP_PREFIX):] - sw_out = os.path.join(self.tmpdir, "stackwalk.log") + sw_out = os.path.join(TEMP_DIR, self.tprefix + self.fault + ".stackwalk.log") with open(sw_out, "a+") as sw_f: print(f"Extracting stack trace to '{sw_out}'...") sw_f.truncate() @@ -104,10 +100,33 @@ def Do(self): TRACE_FUNC = "InjectFaultCmd::Run" self.Verify(TRACE_FUNC in sw, "function names not found in trace (did you build with symbols?)") -passed = (True - & NaclCrashTest("exception").Go() - & NaclCrashTest("throw").Go() - & NaclCrashTest("abort").Go() - & NaclCrashTest("segfault").Go()) +def DoModule(module): + try: + engine = DAEMON_TTYCLIENT + if module == "sgame": + target = os.path.join(GAME_BUILD_DIR, f"sgame-{ARCH}.nexe") + tprefix = "sgame." + elif module == "cgame": + target = os.path.join(GAME_BUILD_DIR, f"cgame-{ARCH}.nexe") + tprefix = "cgame." + elif module == "server": + tprefix = "" + engine = target = DAEMON_SERVER + + assert os.path.isfile(target), target + print(f"Symbolizing '{target}'...") + subprocess.check_call([sys.executable, SYMBOLIZE, "--symbol-directory", SYMBOL_DIR, target]) + + return (True + & NaclCrashTest(DAEMON_TTYCLIENT, tprefix, "exception").Go() + & NaclCrashTest(DAEMON_TTYCLIENT, tprefix, "throw").Go() + & NaclCrashTest(DAEMON_TTYCLIENT, tprefix, "abort").Go() + & NaclCrashTest(DAEMON_TTYCLIENT, tprefix, "segfault").Go()) + except Exception: + raise + return False + +#passed = DoModule("sgame") & DoModule("cgame") +passed = DoModule("server") sys.exit(1 - passed) From c4b6dd33f3e3e8cb10740ea11410d4d893aa554b Mon Sep 17 00:00:00 2001 From: slipher Date: Tue, 2 Sep 2025 16:27:11 -0500 Subject: [PATCH 3/8] windows --- tools/crash_test.py | 204 +++++++++++++++++++++++++++----------------- 1 file changed, 127 insertions(+), 77 deletions(-) diff --git a/tools/crash_test.py b/tools/crash_test.py index 225834982a..ab0e6dcbc8 100644 --- a/tools/crash_test.py +++ b/tools/crash_test.py @@ -1,32 +1,12 @@ #!/usr/bin/env python3 +import argparse import os +import shutil import subprocess import sys import traceback -DAEMON_DIR = '/unv/Unvanquished/daemon' -BREAKPAD_DIR = '/unv/Unvanquished/daemon/libs/breakpad' -GAME_BUILD_DIR = '.' -ARCH = 'amd64' - -SYMBOLIZE = os.path.join(BREAKPAD_DIR, "symbolize.py") -STACKWALK = os.path.join(BREAKPAD_DIR, "src/processor/minidump_stackwalk") - -DAEMON_TTYCLIENT = os.path.join(GAME_BUILD_DIR, 'daemon-tty') -DAEMON_SERVER = os.path.join(GAME_BUILD_DIR, 'daemonded') -TEMP_DIR = os.path.join(GAME_BUILD_DIR, "crashtest-tmp") -os.makedirs(TEMP_DIR, exist_ok=True) -SYMBOL_DIR = os.path.join(TEMP_DIR, "symbols") -os.makedirs(SYMBOL_DIR, exist_ok=True) -HOMEPATH = os.path.join(TEMP_DIR, "homepath") - - - -assert os.path.isfile(DAEMON_SERVER) -assert os.path.isfile(STACKWALK) - - class CrashTest: def __init__(self, name): self.name = name @@ -38,10 +18,13 @@ def Begin(self): def End(self): print(f"==={self.status}: {self.name}===") - def Verify(self, cond, reason): + def Verify(self, cond, reason=None): if not cond: - print(f"FAILURE: {reason}") + if reason is not None: + print(f"FAILURE: {reason}") self.status = "FAILED" + if GIVE_UP: + raise Exception("Giving up on first failure") def Go(self): self.Begin() @@ -53,80 +36,147 @@ def Go(self): self.End() return self.status == "PASSED" -class NaclCrashTest(CrashTest): - def __init__(self, engine, tprefix, fault): - super().__init__(f"nacl.{fault}") +class BreakpadCrashTest(CrashTest): + def __init__(self, module, engine, tprefix, fault): + super().__init__(module + "." + fault) self.engine = engine self.tprefix = tprefix self.fault = fault + self.dir = os.path.join(TEMP_DIR, self.name) + try: + shutil.rmtree(self.dir) + except FileNotFoundError: + pass + os.makedirs(self.dir) def Do(self): print("Running daemon...") - p = subprocess.run([self.engine, "-set", "vm.sgame.type", "1", + p = subprocess.run([self.engine, + "-set", "vm.sgame.type", "1", "-set", "vm.cgame.type", "1", - #"-set", "logs.level.fs", "warning", - "-set", "sv_fps", "1000", - "-set", "common.framerate.max", "0", - #"-set", "server.private", "2", - #"-set", "sv_networkScope", "0", - "-set", "net_enabled", "0", - "-set", "common.framerate.max", "0", - "-pakpath", "/unv/Unvanquished/pkg", - #"-homepath", HOMEPATH, - #"-pakpath", os.path.join(DAEMON_DIR, "pkg"), - #"-set", "fs_basepak", "testdata", - "+devmap plat23", - "+delay 20f echo CRASHTEST_BEGIN", - f"+delay 20f {self.tprefix}injectFault", self.fault, - "+delay 20f echo CRASHTEST_END", - "+delay 40f quit"], - stderr=subprocess.PIPE, check=False) - log = [s.strip() for s in p.stderr.decode("utf8").splitlines()] - i = log.index("CRASHTEST_BEGIN") - #j = log.index("CRASHTEST_END") - j = len(log) - 1 - # TODO expected vs. actual Warn's - DUMP_PREFIX = "Wrote crash dump to " - dumps = [l for l in log if l.startswith(DUMP_PREFIX)] - assert len(dumps) == 1, "Daemon log contains 1 crash dump" - dump = dumps[0][len(DUMP_PREFIX):] - sw_out = os.path.join(TEMP_DIR, self.tprefix + self.fault + ".stackwalk.log") + "-set", "sv_fps", "1000", + "-set", "common.framerate.max", "0", + "-set", "client.errorPopup", "0", + "-set", "server.private", "2", + "-set", "net_enabled", "0", + "-set", "common.framerate.max", "0", + "-set", "cg_navgenOnLoad", "0", + "-homepath", self.dir, + *DAEMON_USER_ARGS, + *["+devmap plat23"] * (self.tprefix == "sgame."), + "+delay 20f echo CRASHTEST_BEGIN", + f"+delay 20f {self.tprefix}injectFault", self.fault, + "+delay 20f echo CRASHTEST_END", + "+delay 40f quit"], + stderr=subprocess.PIPE, check=bool(self.tprefix)) + dumps = os.listdir(os.path.join(self.dir, "crashdump")) + assert len(dumps) == 1, dumps + dump = os.path.join(self.dir, "crashdump", dumps[0]) + sw_out = os.path.join(TEMP_DIR, self.name + "_stackwalk.log") with open(sw_out, "a+") as sw_f: print(f"Extracting stack trace to '{sw_out}'...") sw_f.truncate() - subprocess.run([STACKWALK, dump, SYMBOL_DIR], check=True, stdout=sw_f, stderr=subprocess.STDOUT) + subprocess.run(Virtualize([os.path.join(BREAKPAD_DIR, "src/processor/minidump_stackwalk"), dump, SYMBOL_DIR]), check=True, stdout=sw_f, stderr=subprocess.STDOUT) sw_f.seek(0) sw = sw_f.read() TRACE_FUNC = "InjectFaultCmd::Run" self.Verify(TRACE_FUNC in sw, "function names not found in trace (did you build with symbols?)") -def DoModule(module): - try: - engine = DAEMON_TTYCLIENT +def Virtualize(cmdline): + bin, *args = cmdline + if EXE: + bin2 = bin.replace("\\", "/") + if bin2.startswith("//wsl.localhost/"): + parts = bin2.split("/") + vm = parts[3] + path = "/" + "/".join(parts[4:]) + return ["wsl", "-d", vm, "--", path] + args + if bin.endswith(".py"): + return [sys.executable] + cmdline + return cmdline + +def ModulePath(module): + base = { + "dummyapp" : "dummyapp" + EXE, + "server": "daemonded" + EXE, + "ttyclient": "daemon-tty" + EXE, + "client": "daemon" + EXE, + "sgame": f"sgame-{NACL_ARCH}.nexe", + "cgame": f"cgame-{NACL_ARCH}.nexe", + }[module] + return os.path.join(GAME_BUILD_DIR, base) + +class ModuleCrashTests(CrashTest): + def __init__(self, module, engine=None): + super().__init__(module) + self.engine = engine + + def Do(self): + module = self.name if module == "sgame": - target = os.path.join(GAME_BUILD_DIR, f"sgame-{ARCH}.nexe") + eng = self.engine or "server" tprefix = "sgame." elif module == "cgame": - target = os.path.join(GAME_BUILD_DIR, f"cgame-{ARCH}.nexe") tprefix = "cgame." - elif module == "server": + eng = self.engine or "ttyclient" + else: tprefix = "" - engine = target = DAEMON_SERVER + eng = module + engine = ModulePath(eng) + target = ModulePath(module) assert os.path.isfile(target), target + assert os.path.isfile(engine), engine print(f"Symbolizing '{target}'...") - subprocess.check_call([sys.executable, SYMBOLIZE, "--symbol-directory", SYMBOL_DIR, target]) - - return (True - & NaclCrashTest(DAEMON_TTYCLIENT, tprefix, "exception").Go() - & NaclCrashTest(DAEMON_TTYCLIENT, tprefix, "throw").Go() - & NaclCrashTest(DAEMON_TTYCLIENT, tprefix, "abort").Go() - & NaclCrashTest(DAEMON_TTYCLIENT, tprefix, "segfault").Go()) - except Exception: - raise - return False - -#passed = DoModule("sgame") & DoModule("cgame") -passed = DoModule("server") -sys.exit(1 - passed) + subprocess.check_call(Virtualize([os.path.join(BREAKPAD_DIR, "symbolize.py"), + "--symbol-directory", SYMBOL_DIR, target])) + + self.Verify(BreakpadCrashTest(module, engine, tprefix, "segfault").Go()) + if tprefix or not EXE: + # apparently abort() is caught on Linux but not Windows + self.Verify(BreakpadCrashTest(module, engine, tprefix, "abort").Go()) + if tprefix: + self.Verify(BreakpadCrashTest(module, engine, tprefix, "exception").Go()) + self.Verify(BreakpadCrashTest(module, engine, tprefix, "throw").Go()) + +def ArgParser(usage=None): + ap = argparse.ArgumentParser(usage=usage) + ap.add_argument("--breakpad-dir", type=str, default=BREAKPAD_DIR, help=r"Path to Breakpad repo containing built dump_syms and stackwalk binaries. It may be a \\wsl.localhost\ path on Windows hosts in order to symbolize NaCl.") + ap.add_argument("--give-up", action="store_true", help="Stop after first test failure") + ap.add_argument("--nacl-arch", type=str, choices=["amd64", "i686", "armhf"], default="amd64") # TODO auto-detect? + ap.add_argument("module", nargs="*", choices=[ + "dummyapp", "server", "ttyclient", "client", + "cgame", "ttyclient:cgame", "client:cgame", + "sgame", "server:sgame", "ttyclient:sgame", "client:sgame"]) + return ap + +BREAKPAD_DIR = os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), "../libs/breakpad")) +GAME_BUILD_DIR = '.' # WSL calls rely on relative paths +TEMP_DIR = os.path.join(GAME_BUILD_DIR, "crashtest-tmp") +SYMBOL_DIR = os.path.join(TEMP_DIR, "symbols") +os.makedirs(TEMP_DIR, exist_ok=True) +os.makedirs(SYMBOL_DIR, exist_ok=True) + +if os.name == "nt": + EXE = '.exe' +else: + EXE = "" + +ap = ArgParser(usage=ArgParser().format_usage().rstrip().removeprefix("usage: ") + + " [--daemon-args ARGS...]") +ap.add_argument("--daemon-args", nargs=argparse.REMAINDER, default=[], + help="Extra arguments for Daemon (e.g. -pakpath)") +pa = ap.parse_args(sys.argv[1:]) +BREAKPAD_DIR = pa.breakpad_dir +GIVE_UP = pa.give_up +DAEMON_USER_ARGS = pa.daemon_args +NACL_ARCH = pa.nacl_arch +modules = pa.module or ["server", "ttyclient", "sgame", "cgame"] +passed = True +for module in modules: + passed &= ModuleCrashTests(*module.split(":")[::-1]).Go() + if not passed and GIVE_UP: + break +sys.exit(1 - passed) From 08285bb7b08228b4ac003ca7b498646c73d15917 Mon Sep 17 00:00:00 2001 From: slipher Date: Wed, 3 Sep 2025 07:27:16 -0500 Subject: [PATCH 4/8] release mode --- tools/crash_test.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/tools/crash_test.py b/tools/crash_test.py index ab0e6dcbc8..bddd1240d3 100644 --- a/tools/crash_test.py +++ b/tools/crash_test.py @@ -6,6 +6,7 @@ import subprocess import sys import traceback +import zipfile class CrashTest: def __init__(self, name): @@ -50,10 +51,11 @@ def __init__(self, module, engine, tprefix, fault): os.makedirs(self.dir) def Do(self): + vmtype = "0" if SYMBOL_ZIPS else "1" print("Running daemon...") p = subprocess.run([self.engine, - "-set", "vm.sgame.type", "1", - "-set", "vm.cgame.type", "1", + "-set", "vm.sgame.type", vmtype, + "-set", "vm.cgame.type", vmtype, "-set", "sv_fps", "1000", "-set", "common.framerate.max", "0", "-set", "client.errorPopup", "0", @@ -123,13 +125,14 @@ def Do(self): tprefix = "" eng = module engine = ModulePath(eng) - target = ModulePath(module) - - assert os.path.isfile(target), target assert os.path.isfile(engine), engine - print(f"Symbolizing '{target}'...") - subprocess.check_call(Virtualize([os.path.join(BREAKPAD_DIR, "symbolize.py"), - "--symbol-directory", SYMBOL_DIR, target])) + + if not SYMBOL_ZIPS: + target = ModulePath(module) + assert os.path.isfile(target), target + print(f"Symbolizing '{target}'...") + subprocess.check_call(Virtualize([os.path.join(BREAKPAD_DIR, "symbolize.py"), + "--symbol-directory", SYMBOL_DIR, target])) self.Verify(BreakpadCrashTest(module, engine, tprefix, "segfault").Go()) if tprefix or not EXE: @@ -144,8 +147,9 @@ def ArgParser(usage=None): ap.add_argument("--breakpad-dir", type=str, default=BREAKPAD_DIR, help=r"Path to Breakpad repo containing built dump_syms and stackwalk binaries. It may be a \\wsl.localhost\ path on Windows hosts in order to symbolize NaCl.") ap.add_argument("--give-up", action="store_true", help="Stop after first test failure") ap.add_argument("--nacl-arch", type=str, choices=["amd64", "i686", "armhf"], default="amd64") # TODO auto-detect? - ap.add_argument("module", nargs="*", choices=[ - "dummyapp", "server", "ttyclient", "client", + ap.add_argument("module", nargs="*", + default="server", # bogus default needed due to buggy argparse + choices=["dummyapp", "server", "ttyclient", "client", "cgame", "ttyclient:cgame", "client:cgame", "sgame", "server:sgame", "ttyclient:sgame", "client:sgame"]) return ap @@ -165,7 +169,7 @@ def ArgParser(usage=None): EXE = "" ap = ArgParser(usage=ArgParser().format_usage().rstrip().removeprefix("usage: ") - + " [--daemon-args ARGS...]") + + " [--daemon-args ARGS...]") ap.add_argument("--daemon-args", nargs=argparse.REMAINDER, default=[], help="Extra arguments for Daemon (e.g. -pakpath)") pa = ap.parse_args(sys.argv[1:]) @@ -173,7 +177,19 @@ def ArgParser(usage=None): GIVE_UP = pa.give_up DAEMON_USER_ARGS = pa.daemon_args NACL_ARCH = pa.nacl_arch -modules = pa.module or ["server", "ttyclient", "sgame", "cgame"] +SYMBOL_ZIPS = [p for p in os.listdir(GAME_BUILD_DIR) if p.startswith("symbols") and p.endswith(".zip")] +modules = pa.module +if isinstance(modules, str): + modules = ["server", "ttyclient", "sgame", "cgame"] + +if SYMBOL_ZIPS: + print("Symbol zip(s) detected. Using release validation mode with pre-built symbols") + for z in SYMBOL_ZIPS: + with zipfile.ZipFile(z, 'r') as z: + z.extractall(SYMBOL_DIR) +else: + print("No symbol zip detected. Using end2end Breakpad tooling test mode with dump_syms") + passed = True for module in modules: passed &= ModuleCrashTests(*module.split(":")[::-1]).Go() From 678eaddcb0f42e9af97a08e2277735d654c92d78 Mon Sep 17 00:00:00 2001 From: slipher Date: Wed, 3 Sep 2025 20:18:50 -0500 Subject: [PATCH 5/8] stuff --- tools/crash_test.py | 72 ++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 30 deletions(-) mode change 100644 => 100755 tools/crash_test.py diff --git a/tools/crash_test.py b/tools/crash_test.py old mode 100644 new mode 100755 index bddd1240d3..61e3b51007 --- a/tools/crash_test.py +++ b/tools/crash_test.py @@ -8,7 +8,18 @@ import traceback import zipfile -class CrashTest: +if os.name == "nt": + EXE = '.exe' +else: + EXE = "" + +def PathJoin(*paths): + p = os.path.join(*paths) + if EXE: + p = p.replace("\\", "/") # for WSL + return p + +class Test: def __init__(self, name): self.name = name @@ -37,13 +48,13 @@ def Go(self): self.End() return self.status == "PASSED" -class BreakpadCrashTest(CrashTest): +class BreakpadCrashTest(Test): def __init__(self, module, engine, tprefix, fault): super().__init__(module + "." + fault) self.engine = engine self.tprefix = tprefix self.fault = fault - self.dir = os.path.join(TEMP_DIR, self.name) + self.dir = PathJoin(TEMP_DIR, self.name) try: shutil.rmtree(self.dir) except FileNotFoundError: @@ -71,14 +82,14 @@ def Do(self): "+delay 20f echo CRASHTEST_END", "+delay 40f quit"], stderr=subprocess.PIPE, check=bool(self.tprefix)) - dumps = os.listdir(os.path.join(self.dir, "crashdump")) + dumps = os.listdir(PathJoin(self.dir, "crashdump")) assert len(dumps) == 1, dumps - dump = os.path.join(self.dir, "crashdump", dumps[0]) - sw_out = os.path.join(TEMP_DIR, self.name + "_stackwalk.log") + dump = PathJoin(self.dir, "crashdump", dumps[0]) + sw_out = PathJoin(TEMP_DIR, self.name + "_stackwalk.log") with open(sw_out, "a+") as sw_f: print(f"Extracting stack trace to '{sw_out}'...") sw_f.truncate() - subprocess.run(Virtualize([os.path.join(BREAKPAD_DIR, "src/processor/minidump_stackwalk"), dump, SYMBOL_DIR]), check=True, stdout=sw_f, stderr=subprocess.STDOUT) + subprocess.run(Virtualize([PathJoin(BREAKPAD_DIR, "src/processor/minidump_stackwalk"), dump, SYMBOL_DIR]), check=True, stdout=sw_f, stderr=subprocess.STDOUT) sw_f.seek(0) sw = sw_f.read() TRACE_FUNC = "InjectFaultCmd::Run" @@ -106,9 +117,9 @@ def ModulePath(module): "sgame": f"sgame-{NACL_ARCH}.nexe", "cgame": f"cgame-{NACL_ARCH}.nexe", }[module] - return os.path.join(GAME_BUILD_DIR, base) + return PathJoin(GAME_DIR, base) -class ModuleCrashTests(CrashTest): +class ModuleCrashTests(Test): def __init__(self, module, engine=None): super().__init__(module) self.engine = engine @@ -131,7 +142,7 @@ def Do(self): target = ModulePath(module) assert os.path.isfile(target), target print(f"Symbolizing '{target}'...") - subprocess.check_call(Virtualize([os.path.join(BREAKPAD_DIR, "symbolize.py"), + subprocess.check_call(Virtualize([PathJoin(BREAKPAD_DIR, "symbolize.py"), "--symbol-directory", SYMBOL_DIR, target])) self.Verify(BreakpadCrashTest(module, engine, tprefix, "segfault").Go()) @@ -143,52 +154,53 @@ def Do(self): self.Verify(BreakpadCrashTest(module, engine, tprefix, "throw").Go()) def ArgParser(usage=None): - ap = argparse.ArgumentParser(usage=usage) + ap = argparse.ArgumentParser( + usage=usage, + description="Verify that Breakpad toolchain can produce usable stack traces." + " A Daemon build must be found in the current directory. Also Breakpad's tools must be built in its source tree." + " If a symbols zip is found in the current directory, enter release validation mode: prebuilt symbols are used and VM type defaults to 0 (NaCl from paks)." + " Otherwise, enter end-to-end mode: symbols are produced from the binaries and VM type defaults to 1 (NaCl from PWD). In this mode you will likely need to provide pak paths via --daemon-args.") + ap.add_argument("--game-dir", type=str, default=".", help="Path to Daemon (+ gamelogic) binaries") ap.add_argument("--breakpad-dir", type=str, default=BREAKPAD_DIR, help=r"Path to Breakpad repo containing built dump_syms and stackwalk binaries. It may be a \\wsl.localhost\ path on Windows hosts in order to symbolize NaCl.") ap.add_argument("--give-up", action="store_true", help="Stop after first test failure") ap.add_argument("--nacl-arch", type=str, choices=["amd64", "i686", "armhf"], default="amd64") # TODO auto-detect? ap.add_argument("module", nargs="*", default="server", # bogus default needed due to buggy argparse choices=["dummyapp", "server", "ttyclient", "client", - "cgame", "ttyclient:cgame", "client:cgame", - "sgame", "server:sgame", "ttyclient:sgame", "client:sgame"]) + "cgame", "ttyclient:cgame", "client:cgame", + "sgame", "server:sgame", "ttyclient:sgame", "client:sgame"]) return ap -BREAKPAD_DIR = os.path.abspath(os.path.join( +BREAKPAD_DIR = os.path.abspath(PathJoin( os.path.dirname(os.path.realpath(__file__)), "../libs/breakpad")) -GAME_BUILD_DIR = '.' # WSL calls rely on relative paths -TEMP_DIR = os.path.join(GAME_BUILD_DIR, "crashtest-tmp") -SYMBOL_DIR = os.path.join(TEMP_DIR, "symbols") - -os.makedirs(TEMP_DIR, exist_ok=True) -os.makedirs(SYMBOL_DIR, exist_ok=True) -if os.name == "nt": - EXE = '.exe' -else: - EXE = "" - -ap = ArgParser(usage=ArgParser().format_usage().rstrip().removeprefix("usage: ") - + " [--daemon-args ARGS...]") +ap = ArgParser( + usage=ArgParser().format_usage().rstrip().removeprefix("usage: ") + " [--daemon-args ARGS...]") ap.add_argument("--daemon-args", nargs=argparse.REMAINDER, default=[], help="Extra arguments for Daemon (e.g. -pakpath)") pa = ap.parse_args(sys.argv[1:]) +GAME_DIR = pa.game_dir BREAKPAD_DIR = pa.breakpad_dir GIVE_UP = pa.give_up DAEMON_USER_ARGS = pa.daemon_args NACL_ARCH = pa.nacl_arch -SYMBOL_ZIPS = [p for p in os.listdir(GAME_BUILD_DIR) if p.startswith("symbols") and p.endswith(".zip")] +SYMBOL_ZIPS = [p for p in os.listdir(GAME_DIR) if p.startswith("symbols") and p.endswith(".zip")] modules = pa.module if isinstance(modules, str): modules = ["server", "ttyclient", "sgame", "cgame"] +TEMP_DIR = "crashtest-tmp" # WSL relies on this being relative +SYMBOL_DIR = PathJoin(TEMP_DIR, "symbols") +os.makedirs(TEMP_DIR, exist_ok=True) +os.makedirs(SYMBOL_DIR, exist_ok=True) + if SYMBOL_ZIPS: print("Symbol zip(s) detected. Using release validation mode with pre-built symbols") for z in SYMBOL_ZIPS: - with zipfile.ZipFile(z, 'r') as z: + with zipfile.ZipFile(PathJoin(GAME_DIR, z), 'r') as z: z.extractall(SYMBOL_DIR) else: - print("No symbol zip detected. Using end2end Breakpad tooling test mode with dump_syms") + print("No symbol zip detected. Using end-to-end Breakpad tooling test mode with dump_syms") passed = True for module in modules: From 6726faf1422e170b4f2fe97092926ca5c50bf3c6 Mon Sep 17 00:00:00 2001 From: slipher Date: Thu, 4 Sep 2025 19:15:42 -0500 Subject: [PATCH 6/8] check crash_server --- tools/crash_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/crash_test.py b/tools/crash_test.py index 61e3b51007..1ac97be527 100755 --- a/tools/crash_test.py +++ b/tools/crash_test.py @@ -135,6 +135,7 @@ def Do(self): else: tprefix = "" eng = module + assert os.path.isfile(PathJoin(GAME_DIR, "crash_server" + EXE)) engine = ModulePath(eng) assert os.path.isfile(engine), engine From c2a93710c96e168690adc01be749a3a7f011a50d Mon Sep 17 00:00:00 2001 From: slipher Date: Fri, 5 Sep 2025 17:19:10 -0500 Subject: [PATCH 7/8] function+file names --- tools/crash_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/crash_test.py b/tools/crash_test.py index 1ac97be527..091ee7e478 100755 --- a/tools/crash_test.py +++ b/tools/crash_test.py @@ -92,8 +92,10 @@ def Do(self): subprocess.run(Virtualize([PathJoin(BREAKPAD_DIR, "src/processor/minidump_stackwalk"), dump, SYMBOL_DIR]), check=True, stdout=sw_f, stderr=subprocess.STDOUT) sw_f.seek(0) sw = sw_f.read() - TRACE_FUNC = "InjectFaultCmd::Run" - self.Verify(TRACE_FUNC in sw, "function names not found in trace (did you build with symbols?)") + # Check both function names and filenames. With the Unvanquished 0.55.2 release on Linux, + # we get file:line info but not function names... + self.Verify("CommandSystem.cpp" in sw, "source file names not found in trace") + self.Verify("InjectFaultCmd::Run" in sw, "function names not found in trace") def Virtualize(cmdline): bin, *args = cmdline From 80df5e8769041eca4c3c288e1b8076834d0c9d35 Mon Sep 17 00:00:00 2001 From: slipher Date: Mon, 13 Oct 2025 09:15:46 -0500 Subject: [PATCH 8/8] crash_test adjustmetns --- tools/crash_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/crash_test.py b/tools/crash_test.py index 091ee7e478..a8089a40a6 100755 --- a/tools/crash_test.py +++ b/tools/crash_test.py @@ -63,10 +63,11 @@ def __init__(self, module, engine, tprefix, fault): def Do(self): vmtype = "0" if SYMBOL_ZIPS else "1" - print("Running daemon...") + print("Running Daemon...") p = subprocess.run([self.engine, "-set", "vm.sgame.type", vmtype, "-set", "vm.cgame.type", vmtype, + "-set", "vm.timeout", "500", "-set", "sv_fps", "1000", "-set", "common.framerate.max", "0", "-set", "client.errorPopup", "0", @@ -92,9 +93,8 @@ def Do(self): subprocess.run(Virtualize([PathJoin(BREAKPAD_DIR, "src/processor/minidump_stackwalk"), dump, SYMBOL_DIR]), check=True, stdout=sw_f, stderr=subprocess.STDOUT) sw_f.seek(0) sw = sw_f.read() - # Check both function names and filenames. With the Unvanquished 0.55.2 release on Linux, - # we get file:line info but not function names... - self.Verify("CommandSystem.cpp" in sw, "source file names not found in trace") + # Check both function names and filenames. On Linux it seems like only one of them works at a time?? + self.Verify("Command.cpp" in sw, "source file names not found in trace") self.Verify("InjectFaultCmd::Run" in sw, "function names not found in trace") def Virtualize(cmdline):