From 7aa78957eebdc91e68399a775a56af4ec4c10645 Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Tue, 4 Nov 2025 16:52:53 +0100 Subject: [PATCH 01/10] Add test scripted builder Add annotated builder Test of Polly builder run fix Don't clobber twice --- .buildbot/common/util.py | 253 ++++++++++++++++++ .buildbot/common/worker.py | 94 +++++++ .../polly-x86_64-linux-test-suite.cmake | 11 + .../polly-x86_64-linux-test-suite.py | 101 +++++++ 4 files changed, 459 insertions(+) create mode 100644 .buildbot/common/util.py create mode 100644 .buildbot/common/worker.py create mode 100644 polly/.buildbot/polly-x86_64-linux-test-suite.cmake create mode 100644 polly/.buildbot/polly-x86_64-linux-test-suite.py diff --git a/.buildbot/common/util.py b/.buildbot/common/util.py new file mode 100644 index 0000000000000..223375de2c7aa --- /dev/null +++ b/.buildbot/common/util.py @@ -0,0 +1,253 @@ +import errno +import os +import re +import shutil +import subprocess +import sys +from contextlib import contextmanager + + +def clean_dir(path): + """ + Removes directory at path (and all its subdirectories) if it exists, + and creates an empty directory in its place. + """ + try: + rmtree(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + mkdirp(path) + + +def cmake_pjoin(*args): + """ + Join paths like safe_pjoin, but replace backslashes with forward + slashes on platforms where they are path separators. This prevents + CMake from choking when trying to decode what it thinks are escape + sequences in filenames. + """ + result = safe_pjoin(*args) + if os.sep == '\\': + return result.replace('\\', '/') + else: + return result + + +def report(msg): + sys.stderr.write(msg + '\n') + sys.stderr.flush() + + +def report_run_cmd(cmd, shell=False, *args, **kwargs): + """ + Print a command, then executes it using subprocess.check_call. + """ + report('Running: %s' % ((cmd if shell else shquote_cmd(cmd)),)) + sys.stderr.flush() + subprocess.check_call(cmd, shell=shell, *args, **kwargs) + + +def mkdirp(path): + """Create directory path if it does not already exist.""" + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def rmtree(path): + """ + Remove directory path and all its subdirectories. This differs from + shutil.rmtree() in that it tries to adjust permissions so that deletion + will succeed. + """ + # Some files will not be deletable, so we set permissions that allow + # deletion before we try deleting files. + for root, dirs, files in os.walk(path): + os.chmod(root, 0o755) + for f in files: + p = os.path.join(root, f) + os.chmod(p, 0o644) + os.unlink(p) + # At this point, we should have a tree of deletable directories. + shutil.rmtree(path) + + +def safe_pjoin(dirname, *args): + """ + Join path components with os.path.join, skipping the first component + if it is None. + """ + if dirname is None: + return os.path.join(*args) + else: + return os.path.join(dirname, *args) + + +def _shquote_impl(txt, escaped_chars, quoted_chars): + quoted = re.sub(escaped_chars, r'\\\1', txt) + if len(quoted) == len(txt) and not quoted_chars.search(txt): + return txt + else: + return '"' + quoted + '"' + + +_SHQUOTE_POSIX_ESCAPEDCHARS = re.compile(r'(["`$\\])') +_SHQUOTE_POSIX_QUOTEDCHARS = re.compile('[|&;<>()\' \t\n]') + + +def shquote_posix(txt): + """Return txt, appropriately quoted for POSIX shells.""" + return _shquote_impl( + txt, _SHQUOTE_POSIX_ESCAPEDCHARS, _SHQUOTE_POSIX_QUOTEDCHARS) + + +_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') +_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') + + +def shquote_windows(txt): + """Return txt, appropriately quoted for Windows's cmd.exe.""" + return _shquote_impl( + txt.replace('%', '%%'), + _SHQUOTE_WINDOWS_ESCAPEDCHARS, _SHQUOTE_WINDOWS_QUOTEDCHARS) + + +def shquote(txt): + """Return txt, appropriately quoted for use in a shell command.""" + if os.name in set(('nt', 'os2', 'ce')): + return shquote_windows(txt) + else: + return shquote_posix(txt) + + +def shquote_cmd(cmd): + """Convert a list of shell arguments to an appropriately quoted string.""" + return ' '.join(map(shquote, cmd)) + + + +def clean_dir(path): + """ + Removes directory at path (and all its subdirectories) if it exists, + and creates an empty directory in its place. + """ + try: + rmtree(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + mkdirp(path) + + +def cmake_pjoin(*args): + """ + Join paths like safe_pjoin, but replace backslashes with forward + slashes on platforms where they are path separators. This prevents + CMake from choking when trying to decode what it thinks are escape + sequences in filenames. + """ + result = safe_pjoin(*args) + if os.sep == '\\': + return result.replace('\\', '/') + else: + return result + + +def report(msg): + sys.stderr.write(msg + '\n') + sys.stderr.flush() + + +def report_run_cmd(cmd, shell=False, *args, **kwargs): + """ + Print a command, then executes it using subprocess.check_call. + """ + report('Running: %s' % ((cmd if shell else shquote_cmd(cmd)),)) + sys.stderr.flush() + subprocess.check_call([str(c) for c in cmd], shell=shell, *args, **kwargs) + + +def mkdirp(path): + """Create directory path if it does not already exist.""" + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def rmtree(path): + """ + Remove directory path and all its subdirectories. This differs from + shutil.rmtree() in that it tries to adjust permissions so that deletion + will succeed. + """ + # Some files will not be deletable, so we set permissions that allow + # deletion before we try deleting files. + for root, dirs, files in os.walk(path): + os.chmod(root, 0o755) + for f in files: + p = os.path.join(root, f) + os.chmod(p, 0o644) + os.unlink(p) + # At this point, we should have a tree of deletable directories. + shutil.rmtree(path) + + +def safe_pjoin(dirname, *args): + """ + Join path components with os.path.join, skipping the first component + if it is None. + """ + if dirname is None: + return os.path.join(*args) + else: + return os.path.join(dirname, *args) + + +def _shquote_impl(txt, escaped_chars, quoted_chars): + quoted = re.sub(escaped_chars, r'\\\1', txt) + if len(quoted) == len(txt) and not quoted_chars.search(txt): + return txt + else: + return '"' + quoted + '"' + + +_SHQUOTE_POSIX_ESCAPEDCHARS = re.compile(r'(["`$\\])') +_SHQUOTE_POSIX_QUOTEDCHARS = re.compile('[|&;<>()\' \t\n]') + + +def shquote_posix(txt): + """Return txt, appropriately quoted for POSIX shells.""" + return _shquote_impl( + txt, _SHQUOTE_POSIX_ESCAPEDCHARS, _SHQUOTE_POSIX_QUOTEDCHARS) + + +_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') +_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') + + +def shquote_windows(txt): + """Return txt, appropriately quoted for Windows's cmd.exe.""" + return _shquote_impl( + txt.replace('%', '%%'), + _SHQUOTE_WINDOWS_ESCAPEDCHARS, _SHQUOTE_WINDOWS_QUOTEDCHARS) + + +def shquote(txt): + """Return txt, appropriately quoted for use in a shell command.""" + if os.name in set(('nt', 'os2', 'ce')): + return shquote_windows(txt) + else: + return shquote_posix(txt) + + +def shquote_cmd(cmd): + """Convert a list of shell arguments to an appropriately quoted string.""" + return ' '.join(map(shquote, cmd)) + + + diff --git a/.buildbot/common/worker.py b/.buildbot/common/worker.py new file mode 100644 index 0000000000000..f712a588f46f9 --- /dev/null +++ b/.buildbot/common/worker.py @@ -0,0 +1,94 @@ +import argparse +import os +import subprocess +import sys +import traceback +import util +import traceback +import errno +import re +import subprocess +from contextlib import contextmanager + +@contextmanager +def step(step_name, halt_on_fail=False): + util.report('@@@BUILD_STEP {}@@@'.format(step_name)) + if halt_on_fail: + util.report('@@@HALT_ON_FAILURE@@@') + try: + yield + except Exception as e: + if isinstance(e, subprocess.CalledProcessError): + util.report( + '{} exited with return code {}.'.format(e.cmd, e.returncode) + ) + else: + util.report('The build step threw an exception...') + traceback.print_exc() + + util.report('@@@STEP_FAILURE@@@') + if halt_on_fail: + exit(1) + finally: + sys.stdout.flush() + + +def get_steps(makefile): + try: + make_cmd = build_make_cmd(makefile, 'get-steps') + raw_steps = capture_cmd_stdout(make_cmd) + return raw_steps.decode('utf-8').split('\n')[:-1] + except: + return [] + +def build_make_cmd(makefile, target, make_vars={}): + make_cmd = ['make', '-f', makefile] + if not target is None: + make_cmd.append(target) + for k,v in make_vars.items(): + make_cmd += ["{}={}".format(k, v)] + return make_cmd + +def capture_cmd_stdout(cmd, **kwargs): + return subprocess.run(cmd, shell=False, check=True, stdout=subprocess.PIPE, **kwargs).stdout + +def run_command(cmd, **kwargs): + util.report_run_cmd(cmd, **kwargs) + + +def run_ninja(args, targets, **kwargs): + cmd = ['ninja', *targets] + if args.jobs: + args .append(f'-j{args.jobs}') + #env = os.environ.copy() + #env['NINJA_STATUS'] = "[%p/%es :: %u->%r->%f (of %t)] " + util.report_run_cmd(cmd, **kwargs) + + + + +def checkout(giturl, sourcepath): + if not os.path.exists(sourcepath): + run_command(['git', 'clone', giturl, sourcepath]) + + # Reset repository state no matter what there was before + run_command(['git', '-C', sourcepath, 'stash', '--all']) + run_command(['git', '-C', sourcepath, 'stash', 'clear']) + + # Fetch and checkout the newest + run_command(['git', '-C', sourcepath, 'fetch', 'origin']) + run_command(['git', '-C', sourcepath, 'checkout', 'origin/main', '--detach']) + + +def clean_on_request(args, always=[],on_clobber=[],on_clean=[]): + cleanset = always + if args.clobber or args.clean: + # Clean implies clobber + cleanset += on_clobber + if args.clean: + cleanset += on_clean + + for d in cleanset: + with step(f'delete-{os.path.basename(d)}'): + util.clean_dir(d) + diff --git a/polly/.buildbot/polly-x86_64-linux-test-suite.cmake b/polly/.buildbot/polly-x86_64-linux-test-suite.cmake new file mode 100644 index 0000000000000..3c27965c96f97 --- /dev/null +++ b/polly/.buildbot/polly-x86_64-linux-test-suite.cmake @@ -0,0 +1,11 @@ +# General settings +set(CMAKE_BUILD_TYPE Release CACHE STRING "") +set(CMAKE_C_COMPILER_LAUNCHER ccache CACHE STRING "") +set(CMAKE_CXX_COMPILER_LAUNCHER ccache CACHE STRING "") +set(LLVM_ENABLE_LLD ON CACHE BOOL "") + +set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "") +set(LLVM_ENABLE_PROJECTS "clang;polly" CACHE STRING "") +set(LLVM_TARGETS_TO_BUILD "X86" CACHE STRING "") + +set(LLVM_POLLY_LINK_INTO_TOOLS ON CACHE BOOL "") diff --git a/polly/.buildbot/polly-x86_64-linux-test-suite.py b/polly/.buildbot/polly-x86_64-linux-test-suite.py new file mode 100644 index 0000000000000..cff8b386df2bb --- /dev/null +++ b/polly/.buildbot/polly-x86_64-linux-test-suite.py @@ -0,0 +1,101 @@ +#! /usr/bin/python3 + +import os +import sys +import argparse +import pathlib + +llvmsrcroot = os.path.normpath(f"{__file__}/../../..") # Adapt to location in source tree +sys.path.insert(0, os.path.join(llvmsrcroot, '.buildbot/common')) +import worker + + +# For information +for k,v in os.environ.items(): + print(f"{k}={v}") + +def relative_if_possible(path, relative_to): + path = os.path.normpath(path) + try: + result = os.path.relpath(path, start=relative_to) + return result if result else path + except Exception: + return path + + +parser = argparse.ArgumentParser() +parser.add_argument('--cachefile', default=relative_if_possible(pathlib.Path(__file__).with_suffix('.cmake'), llvmsrcroot), help='CMake cache seed') +parser.add_argument('--jobs', '-j', help='Override the number fo default jobs') +parser.add_argument('--clean', type=bool, default=os.environ.get('BUILDBOT_CLEAN'), help='Whether to delete source-, install-, and build-dirs before running') +parser.add_argument('--clobber', type=bool, default=os.environ.get('BUILDBOT_CLOBBER'), help='Whether to delete install- and build-dirs before running') +args, _ = parser.parse_known_args() + + +cwd = os.getcwd() + + + + + +buildbot_buildername = os.environ.get('BUILDBOT_BUILDERNAME') +buildbot_revision = os.environ.get('BUILDBOT_REVISION', 'origin/main') + + +os.environ['NINJA_STATUS'] = "[%p/%es :: %u->%r->%f (of %t)] " + +llvmbuilddir = "build-llvm" +testsuitesrcdir = "testsuite.src" +testsuitebuilddir = "build-testsuite" +llvminstalldir = 'install-llvm' +print(f"Using build directory: {cwd}") + +# NEVER clean llvmsrcroot or cwd! +worker.clean_on_request(args, always=[llvminstalldir,testsuitebuilddir],on_clobber=[llvmbuilddir],on_clean=[testsuitesrcdir]) + + +with worker.step('configure-llvm', halt_on_fail=True): + cmd = ['cmake', + '-S', os.path.join(llvmsrcroot,'llvm'), + '-B', llvmbuilddir, + '-G', 'Ninja', + '-C', os.path.join(llvmsrcroot, args.cachefile), + f'-DCMAKE_INSTALL_PREFIX={llvminstalldir}' + ] + if args.jobs: + cmd.append(f'-DLLVM_LIT_ARGS=-svj{args.jobs}') + worker.run_command(cmd) + +with worker.step('build-llvm', halt_on_fail=True): + worker.run_command(['ninja', '-C', llvmbuilddir]) + +with worker.step('check-polly'): + worker.run_command(['ninja', '-C', llvmbuilddir, 'check-polly']) + +with worker. step('install-llvm', halt_on_fail=True): + worker.run_command(['ninja', '-C', llvmbuilddir, 'install']) + +with worker. step('clone-testsuite', halt_on_fail=True): + worker.checkout('https://github.com/llvm/llvm-test-suite',testsuitesrcdir) + +with worker.step('configure-testsuite', halt_on_fail=True): + cmd = ['cmake', + '-S', testsuitesrcdir, + '-B', testsuitebuilddir, + '-G', 'Ninja', + '-C', os.path.join(llvmsrcroot, args.cachefile), + '-DCMAKE_BUILD_TYPE=Release', + f'-DCMAKE_C_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang', + f'-DCMAKE_CXX_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang++', + f'-DCMAKE_C_FLAGS=-mllvm -polly', + f'-DCMAKE_CXX_FLAGS=-mllvm -polly', + ] + if args.jobs: + cmd.append(f'-DLLVM_LIT_ARGS=-svj{args.jobs}') + worker.run_command(cmd) + +with worker.step('build-testsuite', halt_on_fail=True): + worker. run_ninja(args, ['-C', testsuitebuilddir]) + +with worker.step('check-testsuite'): + worker.run_ninja(args, ['-C', testsuitebuilddir, 'check']) + From 6a423c186c4b6d0215bf2a3b5d0f440ed72e5778 Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Fri, 7 Nov 2025 00:42:57 +0100 Subject: [PATCH 02/10] Ignore if built in cwd --- polly/.buildbot/.gitignore | 4 ++++ polly/.buildbot/polly-x86_64-linux-test-suite.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 polly/.buildbot/.gitignore diff --git a/polly/.buildbot/.gitignore b/polly/.buildbot/.gitignore new file mode 100644 index 0000000000000..cc083ee5e7a6a --- /dev/null +++ b/polly/.buildbot/.gitignore @@ -0,0 +1,4 @@ +/build-llvm +/install-llvm +/testsuite.src +/build-testsuite diff --git a/polly/.buildbot/polly-x86_64-linux-test-suite.py b/polly/.buildbot/polly-x86_64-linux-test-suite.py index cff8b386df2bb..6ed48b5dc612a 100644 --- a/polly/.buildbot/polly-x86_64-linux-test-suite.py +++ b/polly/.buildbot/polly-x86_64-linux-test-suite.py @@ -5,7 +5,9 @@ import argparse import pathlib -llvmsrcroot = os.path.normpath(f"{__file__}/../../..") # Adapt to location in source tree +# Adapt to location in source tree +llvmsrcroot = os.path.normpath(f"{__file__}/../../..") + sys.path.insert(0, os.path.join(llvmsrcroot, '.buildbot/common')) import worker From eb37650942dd26f082a9f12a2be2b5ab6cc22532 Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Fri, 7 Nov 2025 01:05:45 +0100 Subject: [PATCH 03/10] test script update --- .buildbot/common/worker.py | 30 +++++++++++++++++-- .../polly-x86_64-linux-test-suite.py | 30 ++++++------------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/.buildbot/common/worker.py b/.buildbot/common/worker.py index f712a588f46f9..99bfea9a66bdf 100644 --- a/.buildbot/common/worker.py +++ b/.buildbot/common/worker.py @@ -5,11 +5,37 @@ import traceback import util import traceback -import errno -import re import subprocess from contextlib import contextmanager +def relative_if_possible(path, relative_to): + path = os.path.normpath(path) + try: + result = os.path.relpath(path, start=relative_to) + return result if result else path + except Exception: + return path + + +def common_init(jobs=None): + # For information + for k,v in os.environ.items(): + print(f"{k}={v}") + + jobs_default = None + if jobsenv := os.environ.get('BUILDBOT_JOBS'): + jobs_default = int(jobsenv) + if not jobs_default: + jobs_default = jobs + + parser = argparse.ArgumentParser(allow_abbrev=True, description="Run the buildbot builder configuration; when executed without arguments, builds the llvm-project checkout this file is in, using cwd to write all files") + parser.add_argument('--jobs', '-j', default=jobs_default, help='Override the number fo default jobs') + parser.add_argument('--clean', type=bool, default=os.environ.get('BUILDBOT_CLEAN'), help='Whether to delete source-, build-, and install-dirs (i.e. remove any files leftover from a previous run, if any) before running') + parser.add_argument('--clobber', type=bool, default=os.environ.get('BUILDBOT_CLOBBER'), help='Whether to delete build- and install-dirs before running') + parser.add_argument('--workdir', help="Use this dir as workdir (default: current working directory)" ) + return parser + + @contextmanager def step(step_name, halt_on_fail=False): util.report('@@@BUILD_STEP {}@@@'.format(step_name)) diff --git a/polly/.buildbot/polly-x86_64-linux-test-suite.py b/polly/.buildbot/polly-x86_64-linux-test-suite.py index 6ed48b5dc612a..83e529f65aab8 100644 --- a/polly/.buildbot/polly-x86_64-linux-test-suite.py +++ b/polly/.buildbot/polly-x86_64-linux-test-suite.py @@ -2,7 +2,6 @@ import os import sys -import argparse import pathlib # Adapt to location in source tree @@ -12,26 +11,13 @@ import worker -# For information -for k,v in os.environ.items(): - print(f"{k}={v}") -def relative_if_possible(path, relative_to): - path = os.path.normpath(path) - try: - result = os.path.relpath(path, start=relative_to) - return result if result else path - except Exception: - return path - - -parser = argparse.ArgumentParser() -parser.add_argument('--cachefile', default=relative_if_possible(pathlib.Path(__file__).with_suffix('.cmake'), llvmsrcroot), help='CMake cache seed') -parser.add_argument('--jobs', '-j', help='Override the number fo default jobs') -parser.add_argument('--clean', type=bool, default=os.environ.get('BUILDBOT_CLEAN'), help='Whether to delete source-, install-, and build-dirs before running') -parser.add_argument('--clobber', type=bool, default=os.environ.get('BUILDBOT_CLOBBER'), help='Whether to delete install- and build-dirs before running') +parser = worker.common_init() +parser.add_argument('--cachefile', default=worker.relative_if_possible(pathlib.Path(__file__).with_suffix('.cmake'), llvmsrcroot), help='CMake cache seed') args, _ = parser.parse_known_args() +if args.workdir: + os.chdir(args.workdir) cwd = os.getcwd() @@ -88,11 +74,13 @@ def relative_if_possible(path, relative_to): '-DCMAKE_BUILD_TYPE=Release', f'-DCMAKE_C_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang', f'-DCMAKE_CXX_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang++', - f'-DCMAKE_C_FLAGS=-mllvm -polly', - f'-DCMAKE_CXX_FLAGS=-mllvm -polly', + f'-DTEST_SUITE_LIT={os.path.abspath(llvmbuilddir)}/bin/llvm-lit', + f'-DTEST_SUITE_LLVM_SIZE={os.path.abspath(llvmbuilddir)}/bin/llvm-size', + "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", + "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", ] if args.jobs: - cmd.append(f'-DLLVM_LIT_ARGS=-svj{args.jobs}') + cmd.append(f'-DLLVM_LIT_ARGS=-svj{args.jobs};-o;report.json') worker.run_command(cmd) with worker.step('build-testsuite', halt_on_fail=True): From 689e4ac341ec0e17502eb309d2a7648f0710fc42 Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Fri, 7 Nov 2025 01:24:04 +0100 Subject: [PATCH 04/10] flush before step-begin marker --- .buildbot/common/worker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildbot/common/worker.py b/.buildbot/common/worker.py index 99bfea9a66bdf..3f084ecd01dc7 100644 --- a/.buildbot/common/worker.py +++ b/.buildbot/common/worker.py @@ -38,6 +38,8 @@ def common_init(jobs=None): @contextmanager def step(step_name, halt_on_fail=False): + sys.stderr.flush() + sys.stdout.flush() util.report('@@@BUILD_STEP {}@@@'.format(step_name)) if halt_on_fail: util.report('@@@HALT_ON_FAILURE@@@') @@ -55,8 +57,7 @@ def step(step_name, halt_on_fail=False): util.report('@@@STEP_FAILURE@@@') if halt_on_fail: exit(1) - finally: - sys.stdout.flush() + def get_steps(makefile): From 76cde342aec5cc07cfc7521ff3447c1055af418f Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Fri, 7 Nov 2025 01:31:33 +0100 Subject: [PATCH 05/10] Emit ccache-stats --- .buildbot/common/worker.py | 9 +++++---- polly/.buildbot/polly-x86_64-linux-test-suite.cmake | 6 +++--- polly/.buildbot/polly-x86_64-linux-test-suite.py | 12 ++++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.buildbot/common/worker.py b/.buildbot/common/worker.py index 3f084ecd01dc7..2f657a77f4b9e 100644 --- a/.buildbot/common/worker.py +++ b/.buildbot/common/worker.py @@ -83,14 +83,15 @@ def run_command(cmd, **kwargs): util.report_run_cmd(cmd, **kwargs) -def run_ninja(args, targets, **kwargs): +def run_ninja(args, targets, ccache_stats=False, **kwargs): cmd = ['ninja', *targets] if args.jobs: args .append(f'-j{args.jobs}') - #env = os.environ.copy() - #env['NINJA_STATUS'] = "[%p/%es :: %u->%r->%f (of %t)] " + if ccache_stats: + util.report_run_cmd(['ccache', '-z']) util.report_run_cmd(cmd, **kwargs) - + if ccache_stats: + util.report_run_cmd(['ccache', '-sv']) diff --git a/polly/.buildbot/polly-x86_64-linux-test-suite.cmake b/polly/.buildbot/polly-x86_64-linux-test-suite.cmake index 3c27965c96f97..f5699804f2109 100644 --- a/polly/.buildbot/polly-x86_64-linux-test-suite.cmake +++ b/polly/.buildbot/polly-x86_64-linux-test-suite.cmake @@ -1,7 +1,7 @@ # General settings -set(CMAKE_BUILD_TYPE Release CACHE STRING "") -set(CMAKE_C_COMPILER_LAUNCHER ccache CACHE STRING "") -set(CMAKE_CXX_COMPILER_LAUNCHER ccache CACHE STRING "") +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") +set(CMAKE_C_COMPILER_LAUNCHER "ccache" CACHE STRING "") +set(CMAKE_CXX_COMPILER_LAUNCHER "ccache" CACHE STRING "") set(LLVM_ENABLE_LLD ON CACHE BOOL "") set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "") diff --git a/polly/.buildbot/polly-x86_64-linux-test-suite.py b/polly/.buildbot/polly-x86_64-linux-test-suite.py index 83e529f65aab8..95655ad4dabdb 100644 --- a/polly/.buildbot/polly-x86_64-linux-test-suite.py +++ b/polly/.buildbot/polly-x86_64-linux-test-suite.py @@ -54,13 +54,13 @@ worker.run_command(cmd) with worker.step('build-llvm', halt_on_fail=True): - worker.run_command(['ninja', '-C', llvmbuilddir]) + worker.run_ninja(args, ['-C', llvmbuilddir], ccache_stats=True) with worker.step('check-polly'): - worker.run_command(['ninja', '-C', llvmbuilddir, 'check-polly']) + worker.run_ninja(args, ['-C', llvmbuilddir, 'check-polly'], ccache_stats=True) -with worker. step('install-llvm', halt_on_fail=True): - worker.run_command(['ninja', '-C', llvmbuilddir, 'install']) +with worker.step('install-llvm', halt_on_fail=True): + worker.run_ninja(args, ['-C', llvmbuilddir, 'install'], ccache_stats=True) with worker. step('clone-testsuite', halt_on_fail=True): worker.checkout('https://github.com/llvm/llvm-test-suite',testsuitesrcdir) @@ -84,8 +84,8 @@ worker.run_command(cmd) with worker.step('build-testsuite', halt_on_fail=True): - worker. run_ninja(args, ['-C', testsuitebuilddir]) + worker. run_ninja(args, ['-C', testsuitebuilddir], ccache_stats=True) with worker.step('check-testsuite'): - worker.run_ninja(args, ['-C', testsuitebuilddir, 'check']) + worker.run_ninja(args, ['-C', testsuitebuilddir, 'check'], ccache_stats=True) From 2d928562d0d9f7c8bde70748d35d8e0bd8fa6a9e Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Fri, 7 Nov 2025 13:01:02 +0100 Subject: [PATCH 06/10] Summary test Rework common logic No ccache stats for checks indention fix diagnostic print Again print debugging Print cwd remove debg --- .buildbot/common/util.py | 253 ----------- .buildbot/common/worker.py | 408 ++++++++++++++++-- polly/.buildbot/.gitignore | 5 +- .../polly-x86_64-linux-test-suite.py | 125 +++--- 4 files changed, 424 insertions(+), 367 deletions(-) delete mode 100644 .buildbot/common/util.py diff --git a/.buildbot/common/util.py b/.buildbot/common/util.py deleted file mode 100644 index 223375de2c7aa..0000000000000 --- a/.buildbot/common/util.py +++ /dev/null @@ -1,253 +0,0 @@ -import errno -import os -import re -import shutil -import subprocess -import sys -from contextlib import contextmanager - - -def clean_dir(path): - """ - Removes directory at path (and all its subdirectories) if it exists, - and creates an empty directory in its place. - """ - try: - rmtree(path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - mkdirp(path) - - -def cmake_pjoin(*args): - """ - Join paths like safe_pjoin, but replace backslashes with forward - slashes on platforms where they are path separators. This prevents - CMake from choking when trying to decode what it thinks are escape - sequences in filenames. - """ - result = safe_pjoin(*args) - if os.sep == '\\': - return result.replace('\\', '/') - else: - return result - - -def report(msg): - sys.stderr.write(msg + '\n') - sys.stderr.flush() - - -def report_run_cmd(cmd, shell=False, *args, **kwargs): - """ - Print a command, then executes it using subprocess.check_call. - """ - report('Running: %s' % ((cmd if shell else shquote_cmd(cmd)),)) - sys.stderr.flush() - subprocess.check_call(cmd, shell=shell, *args, **kwargs) - - -def mkdirp(path): - """Create directory path if it does not already exist.""" - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - -def rmtree(path): - """ - Remove directory path and all its subdirectories. This differs from - shutil.rmtree() in that it tries to adjust permissions so that deletion - will succeed. - """ - # Some files will not be deletable, so we set permissions that allow - # deletion before we try deleting files. - for root, dirs, files in os.walk(path): - os.chmod(root, 0o755) - for f in files: - p = os.path.join(root, f) - os.chmod(p, 0o644) - os.unlink(p) - # At this point, we should have a tree of deletable directories. - shutil.rmtree(path) - - -def safe_pjoin(dirname, *args): - """ - Join path components with os.path.join, skipping the first component - if it is None. - """ - if dirname is None: - return os.path.join(*args) - else: - return os.path.join(dirname, *args) - - -def _shquote_impl(txt, escaped_chars, quoted_chars): - quoted = re.sub(escaped_chars, r'\\\1', txt) - if len(quoted) == len(txt) and not quoted_chars.search(txt): - return txt - else: - return '"' + quoted + '"' - - -_SHQUOTE_POSIX_ESCAPEDCHARS = re.compile(r'(["`$\\])') -_SHQUOTE_POSIX_QUOTEDCHARS = re.compile('[|&;<>()\' \t\n]') - - -def shquote_posix(txt): - """Return txt, appropriately quoted for POSIX shells.""" - return _shquote_impl( - txt, _SHQUOTE_POSIX_ESCAPEDCHARS, _SHQUOTE_POSIX_QUOTEDCHARS) - - -_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') -_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') - - -def shquote_windows(txt): - """Return txt, appropriately quoted for Windows's cmd.exe.""" - return _shquote_impl( - txt.replace('%', '%%'), - _SHQUOTE_WINDOWS_ESCAPEDCHARS, _SHQUOTE_WINDOWS_QUOTEDCHARS) - - -def shquote(txt): - """Return txt, appropriately quoted for use in a shell command.""" - if os.name in set(('nt', 'os2', 'ce')): - return shquote_windows(txt) - else: - return shquote_posix(txt) - - -def shquote_cmd(cmd): - """Convert a list of shell arguments to an appropriately quoted string.""" - return ' '.join(map(shquote, cmd)) - - - -def clean_dir(path): - """ - Removes directory at path (and all its subdirectories) if it exists, - and creates an empty directory in its place. - """ - try: - rmtree(path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - mkdirp(path) - - -def cmake_pjoin(*args): - """ - Join paths like safe_pjoin, but replace backslashes with forward - slashes on platforms where they are path separators. This prevents - CMake from choking when trying to decode what it thinks are escape - sequences in filenames. - """ - result = safe_pjoin(*args) - if os.sep == '\\': - return result.replace('\\', '/') - else: - return result - - -def report(msg): - sys.stderr.write(msg + '\n') - sys.stderr.flush() - - -def report_run_cmd(cmd, shell=False, *args, **kwargs): - """ - Print a command, then executes it using subprocess.check_call. - """ - report('Running: %s' % ((cmd if shell else shquote_cmd(cmd)),)) - sys.stderr.flush() - subprocess.check_call([str(c) for c in cmd], shell=shell, *args, **kwargs) - - -def mkdirp(path): - """Create directory path if it does not already exist.""" - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - -def rmtree(path): - """ - Remove directory path and all its subdirectories. This differs from - shutil.rmtree() in that it tries to adjust permissions so that deletion - will succeed. - """ - # Some files will not be deletable, so we set permissions that allow - # deletion before we try deleting files. - for root, dirs, files in os.walk(path): - os.chmod(root, 0o755) - for f in files: - p = os.path.join(root, f) - os.chmod(p, 0o644) - os.unlink(p) - # At this point, we should have a tree of deletable directories. - shutil.rmtree(path) - - -def safe_pjoin(dirname, *args): - """ - Join path components with os.path.join, skipping the first component - if it is None. - """ - if dirname is None: - return os.path.join(*args) - else: - return os.path.join(dirname, *args) - - -def _shquote_impl(txt, escaped_chars, quoted_chars): - quoted = re.sub(escaped_chars, r'\\\1', txt) - if len(quoted) == len(txt) and not quoted_chars.search(txt): - return txt - else: - return '"' + quoted + '"' - - -_SHQUOTE_POSIX_ESCAPEDCHARS = re.compile(r'(["`$\\])') -_SHQUOTE_POSIX_QUOTEDCHARS = re.compile('[|&;<>()\' \t\n]') - - -def shquote_posix(txt): - """Return txt, appropriately quoted for POSIX shells.""" - return _shquote_impl( - txt, _SHQUOTE_POSIX_ESCAPEDCHARS, _SHQUOTE_POSIX_QUOTEDCHARS) - - -_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') -_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') - - -def shquote_windows(txt): - """Return txt, appropriately quoted for Windows's cmd.exe.""" - return _shquote_impl( - txt.replace('%', '%%'), - _SHQUOTE_WINDOWS_ESCAPEDCHARS, _SHQUOTE_WINDOWS_QUOTEDCHARS) - - -def shquote(txt): - """Return txt, appropriately quoted for use in a shell command.""" - if os.name in set(('nt', 'os2', 'ce')): - return shquote_windows(txt) - else: - return shquote_posix(txt) - - -def shquote_cmd(cmd): - """Convert a list of shell arguments to an appropriately quoted string.""" - return ' '.join(map(shquote, cmd)) - - - diff --git a/.buildbot/common/worker.py b/.buildbot/common/worker.py index 2f657a77f4b9e..f40f0eb4d0865 100644 --- a/.buildbot/common/worker.py +++ b/.buildbot/common/worker.py @@ -1,13 +1,255 @@ import argparse +import filecmp +import tempfile import os import subprocess import sys import traceback -import util -import traceback +import pathlib +import subprocess +import errno +import os +import re +import shutil import subprocess +import sys from contextlib import contextmanager + + + + + +def cmake_pjoin(*args): + """ + Join paths like safe_pjoin, but replace backslashes with forward + slashes on platforms where they are path separators. This prevents + CMake from choking when trying to decode what it thinks are escape + sequences in filenames. + """ + result = safe_pjoin(*args) + if os.sep == '\\': + return result.replace('\\', '/') + else: + return result + + +def report(msg): + sys.stderr.write(msg + '\n') + sys.stderr.flush() + + +def report_run_cmd(cmd, shell=False, *args, **kwargs): + """ + Print a command, then executes it using subprocess.check_call. + """ + report('Running: %s' % ((cmd if shell else shquote_cmd(cmd)),)) + sys.stderr.flush() + subprocess.check_call(cmd, shell=shell, *args, **kwargs) + + +def mkdirp(path): + """Create directory path if it does not already exist.""" + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def rmtree(path): + """ + Remove directory path and all its subdirectories. This differs from + shutil.rmtree() in that it tries to adjust permissions so that deletion + will succeed. + """ + # Some files will not be deletable, so we set permissions that allow + # deletion before we try deleting files. + for root, dirs, files in os.walk(path): + os.chmod(root, 0o755) + for f in files: + p = os.path.join(root, f) + os.chmod(p, 0o644) + os.unlink(p) + # At this point, we should have a tree of deletable directories. + shutil.rmtree(path) + + +def safe_pjoin(dirname, *args): + """ + Join path components with os.path.join, skipping the first component + if it is None. + """ + if dirname is None: + return os.path.join(*args) + else: + return os.path.join(dirname, *args) + + +def _shquote_impl(txt, escaped_chars, quoted_chars): + quoted = re.sub(escaped_chars, r'\\\1', txt) + if len(quoted) == len(txt) and not quoted_chars.search(txt): + return txt + else: + return '"' + quoted + '"' + + +_SHQUOTE_POSIX_ESCAPEDCHARS = re.compile(r'(["`$\\])') +_SHQUOTE_POSIX_QUOTEDCHARS = re.compile('[|&;<>()\' \t\n]') + + +def shquote_posix(txt): + """Return txt, appropriately quoted for POSIX shells.""" + return _shquote_impl( + txt, _SHQUOTE_POSIX_ESCAPEDCHARS, _SHQUOTE_POSIX_QUOTEDCHARS) + + +_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') +_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') + + +def shquote_windows(txt): + """Return txt, appropriately quoted for Windows's cmd.exe.""" + return _shquote_impl( + txt.replace('%', '%%'), + _SHQUOTE_WINDOWS_ESCAPEDCHARS, _SHQUOTE_WINDOWS_QUOTEDCHARS) + + +def shquote(txt): + """Return txt, appropriately quoted for use in a shell command.""" + if os.name in set(('nt', 'os2', 'ce')): + return shquote_windows(txt) + else: + return shquote_posix(txt) + + +def shquote_cmd(cmd): + """Convert a list of shell arguments to an appropriately quoted string.""" + return ' '.join(map(shquote, cmd)) + + + +def clean_dir(path): + """ + Removes directory at path (and all its subdirectories) if it exists, + and creates an empty directory in its place. + """ + try: + rmtree(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + mkdirp(path) + + +def cmake_pjoin(*args): + """ + Join paths like safe_pjoin, but replace backslashes with forward + slashes on platforms where they are path separators. This prevents + CMake from choking when trying to decode what it thinks are escape + sequences in filenames. + """ + result = safe_pjoin(*args) + if os.sep == '\\': + return result.replace('\\', '/') + else: + return result + + +def report(msg): + sys.stderr.write(msg + '\n') + sys.stderr.flush() + + +def report_run_cmd(cmd, shell=False, *args, **kwargs): + """ + Print a command, then executes it using subprocess.check_call. + """ + report('Running: %s' % ((cmd if shell else shquote_cmd(cmd)),)) + sys.stderr.flush() + subprocess.check_call([str(c) for c in cmd], shell=shell, *args, **kwargs) + + +def mkdirp(path): + """Create directory path if it does not already exist.""" + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def rmtree(path): + """ + Remove directory path and all its subdirectories. This differs from + shutil.rmtree() in that it tries to adjust permissions so that deletion + will succeed. + """ + # Some files will not be deletable, so we set permissions that allow + # deletion before we try deleting files. + for root, dirs, files in os.walk(path): + os.chmod(root, 0o755) + for f in files: + p = os.path.join(root, f) + os.chmod(p, 0o644) + os.unlink(p) + # At this point, we should have a tree of deletable directories. + shutil.rmtree(path) + + +def safe_pjoin(dirname, *args): + """ + Join path components with os.path.join, skipping the first component + if it is None. + """ + if dirname is None: + return os.path.join(*args) + else: + return os.path.join(dirname, *args) + + +def _shquote_impl(txt, escaped_chars, quoted_chars): + quoted = re.sub(escaped_chars, r'\\\1', txt) + if len(quoted) == len(txt) and not quoted_chars.search(txt): + return txt + else: + return '"' + quoted + '"' + + +_SHQUOTE_POSIX_ESCAPEDCHARS = re.compile(r'(["`$\\])') +_SHQUOTE_POSIX_QUOTEDCHARS = re.compile('[|&;<>()\' \t\n]') + + +def shquote_posix(txt): + """Return txt, appropriately quoted for POSIX shells.""" + return _shquote_impl( + txt, _SHQUOTE_POSIX_ESCAPEDCHARS, _SHQUOTE_POSIX_QUOTEDCHARS) + + +_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') +_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') + + +def shquote_windows(txt): + """Return txt, appropriately quoted for Windows's cmd.exe.""" + return _shquote_impl( + txt.replace('%', '%%'), + _SHQUOTE_WINDOWS_ESCAPEDCHARS, _SHQUOTE_WINDOWS_QUOTEDCHARS) + + +def shquote(txt): + """Return txt, appropriately quoted for use in a shell command.""" + if os.name in set(('nt', 'os2', 'ce')): + return shquote_windows(txt) + else: + return shquote_posix(txt) + + +def shquote_cmd(cmd): + """Convert a list of shell arguments to an appropriately quoted string.""" + return ' '.join(map(shquote, cmd)) + + def relative_if_possible(path, relative_to): path = os.path.normpath(path) try: @@ -17,49 +259,151 @@ def relative_if_possible(path, relative_to): return path -def common_init(jobs=None): - # For information + + +def first_true(*args): + for a in args: + if a: + return a + return None + + +def first_nonnull(*args): + for a in args: + if a: + return a + return None + + +class Worker: + def __init__(self,args,clean,clobber,workdir,jobs,cachefile,llvmsrcroot): + self.args=args + self.clean =clean + self.clobber=clobber + self.workdir=workdir + self.jobs=jobs + self.cachefile =cachefile + self.llvmsrcroot=llvmsrcroot + + def in_llvmsrc(self, path): + return os.path.join(self.llvmsrcroot, path) + + def in_workdir(self, path): + return os.path.join(self.workdir, path) + + @contextmanager + def step(self, step_name, halt_on_fail=False): + with step(step_name,halt_on_fail=halt_on_fail) as s: + yield s + + def run_cmake(self, cmakeargs): + report_run_cmd(['cmake', *cmakeargs]) + + def run_ninja(args, targets, ccache_stats=False, **kwargs): + cmd = ['ninja', *targets] + if args.jobs: + args .append(f'-j{args.jobs}') + if ccache_stats: + report_run_cmd(['ccache', '-z']) + report_run_cmd(cmd, **kwargs) + if ccache_stats: + report_run_cmd(['ccache', '-sv']) + + def checkout (self,giturl, sourcepath): + return checkout(giturl, sourcepath) + + + + + + +@contextmanager +def run(scriptname, llvmsrcroot, parser=None ,clobberpaths=[], workerjobs=None): for k,v in os.environ.items(): print(f"{k}={v}") - jobs_default = None - if jobsenv := os.environ.get('BUILDBOT_JOBS'): - jobs_default = int(jobsenv) - if not jobs_default: - jobs_default = jobs + os.environ['NINJA_STATUS'] = "[%p/%es :: %u->%r->%f (of %t)] " + + jobs_default = first_true( os.environ.get('BUILDBOT_JOBS') , workerjobs ) + + + + stem = pathlib.Path(scriptname).stem + + + parser = parser or argparse.ArgumentParser(allow_abbrev=True, description="Run the buildbot builder configuration; when executed without arguments, builds the llvm-project checkout this file is in, using cwd to write all files") + parser.add_argument('--jobs', '-j', default=jobs_default, help='Override the number of default jobs') + parser.add_argument('--clean', type=bool, default=os.environ.get('BUILDBOT_CLEAN') , help='Whether to delete source-, build-, and install-dirs (i.e. remove any files leftover from a previous run, if any) before running') + parser.add_argument('--clobber', type=bool, default=os.environ.get('BUILDBOT_CLOBBER') or os.environ.get('BUILDBOT_CLEAN_OBJ'), help='Whether to delete build- and install-dirs before running') + parser.add_argument('--workdir', default=f'{stem}.workdir', help="Use this dir as workdir") + parser.add_argument('--cachefile', default=relative_if_possible(pathlib.Path(scriptname).with_suffix('.cmake'), llvmsrcroot), help='CMake cache seed') + args = parser.parse_args() + + print("cwd:",os.getcwd()) + + workdir = args.workdir + clobber = args.clobber + cachefile = os.path.join(llvmsrcroot, args.cachefile) + oldcwd = os.getcwd() + + + prevcachepath = os.path.join(workdir, 'prevcache.cmake') + if cachefile and os.path.exists(prevcachepath): + # Force clobber if cache file has changed; a new cachefile does not override entries already present in CMakeCache.txt + if not filecmp.cmp( os.path.join(llvmsrcroot, args.cachefile), prevcachepath, shallow=False): + clobber = True + #shutil.copyfile(cachefile) + + w = Worker(args, clean=args.clean , clobber=clobber, workdir=workdir, jobs=args.jobs, cachefile=cachefile, llvmsrcroot=llvmsrcroot) + + #if workdir: + if args.clean: + with w.step(f'clean'): + clean_dir(workdir) + elif clobber: + with w.step(f'clobber'): + for d in clobberpaths: + clean_dir(os.path.join(workdir, d)) + os.path.unlink(prevcachepath) + + os.makedirs(workdir, exist_ok=True) + os.chdir(workdir) + + # Remember used cachefile in case it changes + if cachefile: + shutil.copy2(os.path.join(oldcwd,llvmsrcroot, args.cachefile), os.path.join(oldcwd, prevcachepath )) + + yield w + #else: + # with tempfile.TemporaryDirectory(prefix = stem) as tmpdir: + # os.chdir(tmpdir) + # yield Worker(args) + + + - parser = argparse.ArgumentParser(allow_abbrev=True, description="Run the buildbot builder configuration; when executed without arguments, builds the llvm-project checkout this file is in, using cwd to write all files") - parser.add_argument('--jobs', '-j', default=jobs_default, help='Override the number fo default jobs') - parser.add_argument('--clean', type=bool, default=os.environ.get('BUILDBOT_CLEAN'), help='Whether to delete source-, build-, and install-dirs (i.e. remove any files leftover from a previous run, if any) before running') - parser.add_argument('--clobber', type=bool, default=os.environ.get('BUILDBOT_CLOBBER'), help='Whether to delete build- and install-dirs before running') - parser.add_argument('--workdir', help="Use this dir as workdir (default: current working directory)" ) - return parser @contextmanager def step(step_name, halt_on_fail=False): sys.stderr.flush() sys.stdout.flush() - util.report('@@@BUILD_STEP {}@@@'.format(step_name)) + report(f'@@@BUILD_STEP {step_name}@@@') if halt_on_fail: - util.report('@@@HALT_ON_FAILURE@@@') + report('@@@HALT_ON_FAILURE@@@') try: yield except Exception as e: if isinstance(e, subprocess.CalledProcessError): - util.report( - '{} exited with return code {}.'.format(e.cmd, e.returncode) - ) + report(f'{e.cmd} exited with return code {e.returncode}.' ) + report('@@@STEP_FAILURE@@@') else: - util.report('The build step threw an exception...') traceback.print_exc() - - util.report('@@@STEP_FAILURE@@@') + report('@@@STEP_EXCEPTION@@@') if halt_on_fail: exit(1) - def get_steps(makefile): try: make_cmd = build_make_cmd(makefile, 'get-steps') @@ -68,6 +412,7 @@ def get_steps(makefile): except: return [] + def build_make_cmd(makefile, target, make_vars={}): make_cmd = ['make', '-f', makefile] if not target is None: @@ -76,22 +421,15 @@ def build_make_cmd(makefile, target, make_vars={}): make_cmd += ["{}={}".format(k, v)] return make_cmd + def capture_cmd_stdout(cmd, **kwargs): return subprocess.run(cmd, shell=False, check=True, stdout=subprocess.PIPE, **kwargs).stdout + def run_command(cmd, **kwargs): - util.report_run_cmd(cmd, **kwargs) + report_run_cmd(cmd, **kwargs) -def run_ninja(args, targets, ccache_stats=False, **kwargs): - cmd = ['ninja', *targets] - if args.jobs: - args .append(f'-j{args.jobs}') - if ccache_stats: - util.report_run_cmd(['ccache', '-z']) - util.report_run_cmd(cmd, **kwargs) - if ccache_stats: - util.report_run_cmd(['ccache', '-sv']) @@ -118,5 +456,5 @@ def clean_on_request(args, always=[],on_clobber=[],on_clean=[]): for d in cleanset: with step(f'delete-{os.path.basename(d)}'): - util.clean_dir(d) + clean_dir(d) diff --git a/polly/.buildbot/.gitignore b/polly/.buildbot/.gitignore index cc083ee5e7a6a..f2198f0f4ef8e 100644 --- a/polly/.buildbot/.gitignore +++ b/polly/.buildbot/.gitignore @@ -1,4 +1 @@ -/build-llvm -/install-llvm -/testsuite.src -/build-testsuite +/*.workdir diff --git a/polly/.buildbot/polly-x86_64-linux-test-suite.py b/polly/.buildbot/polly-x86_64-linux-test-suite.py index 95655ad4dabdb..13424833f3a06 100644 --- a/polly/.buildbot/polly-x86_64-linux-test-suite.py +++ b/polly/.buildbot/polly-x86_64-linux-test-suite.py @@ -12,80 +12,55 @@ -parser = worker.common_init() -parser.add_argument('--cachefile', default=worker.relative_if_possible(pathlib.Path(__file__).with_suffix('.cmake'), llvmsrcroot), help='CMake cache seed') -args, _ = parser.parse_known_args() - -if args.workdir: - os.chdir(args.workdir) - -cwd = os.getcwd() - - - - - -buildbot_buildername = os.environ.get('BUILDBOT_BUILDERNAME') -buildbot_revision = os.environ.get('BUILDBOT_REVISION', 'origin/main') - - -os.environ['NINJA_STATUS'] = "[%p/%es :: %u->%r->%f (of %t)] " - -llvmbuilddir = "build-llvm" +llvmbuilddir = "llvm.build" +llvminstalldir = 'llvm.install' testsuitesrcdir = "testsuite.src" -testsuitebuilddir = "build-testsuite" -llvminstalldir = 'install-llvm' -print(f"Using build directory: {cwd}") - -# NEVER clean llvmsrcroot or cwd! -worker.clean_on_request(args, always=[llvminstalldir,testsuitebuilddir],on_clobber=[llvmbuilddir],on_clean=[testsuitesrcdir]) - - -with worker.step('configure-llvm', halt_on_fail=True): - cmd = ['cmake', - '-S', os.path.join(llvmsrcroot,'llvm'), - '-B', llvmbuilddir, - '-G', 'Ninja', - '-C', os.path.join(llvmsrcroot, args.cachefile), - f'-DCMAKE_INSTALL_PREFIX={llvminstalldir}' - ] - if args.jobs: - cmd.append(f'-DLLVM_LIT_ARGS=-svj{args.jobs}') - worker.run_command(cmd) - -with worker.step('build-llvm', halt_on_fail=True): - worker.run_ninja(args, ['-C', llvmbuilddir], ccache_stats=True) - -with worker.step('check-polly'): - worker.run_ninja(args, ['-C', llvmbuilddir, 'check-polly'], ccache_stats=True) - -with worker.step('install-llvm', halt_on_fail=True): - worker.run_ninja(args, ['-C', llvmbuilddir, 'install'], ccache_stats=True) - -with worker. step('clone-testsuite', halt_on_fail=True): - worker.checkout('https://github.com/llvm/llvm-test-suite',testsuitesrcdir) - -with worker.step('configure-testsuite', halt_on_fail=True): - cmd = ['cmake', - '-S', testsuitesrcdir, - '-B', testsuitebuilddir, - '-G', 'Ninja', - '-C', os.path.join(llvmsrcroot, args.cachefile), - '-DCMAKE_BUILD_TYPE=Release', - f'-DCMAKE_C_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang', - f'-DCMAKE_CXX_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang++', - f'-DTEST_SUITE_LIT={os.path.abspath(llvmbuilddir)}/bin/llvm-lit', - f'-DTEST_SUITE_LLVM_SIZE={os.path.abspath(llvmbuilddir)}/bin/llvm-size', - "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", - "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", - ] - if args.jobs: - cmd.append(f'-DLLVM_LIT_ARGS=-svj{args.jobs};-o;report.json') - worker.run_command(cmd) - -with worker.step('build-testsuite', halt_on_fail=True): - worker. run_ninja(args, ['-C', testsuitebuilddir], ccache_stats=True) - -with worker.step('check-testsuite'): - worker.run_ninja(args, ['-C', testsuitebuilddir, 'check'], ccache_stats=True) +testsuitebuilddir = "testsuite.build" + +with worker.run(scriptname=__file__,llvmsrcroot=llvmsrcroot,clobberpaths = [llvmbuilddir,testsuitebuilddir, llvminstalldir]) as w: + with w.step('configure-llvm', halt_on_fail=True): + cmakeargs =[ + '-S', w.in_llvmsrc( 'llvm'), + '-B', llvmbuilddir, + '-G', 'Ninja', + '-C', w.in_llvmsrc( w.cachefile), + f'-DCMAKE_INSTALL_PREFIX={llvminstalldir}' + ] + if w.jobs: + cmakeargs.append( f'-DLLVM_LIT_ARGS=-sv;-j{w.jobs}') + w.run_cmake(cmakeargs) + + with w.step('build-llvm', halt_on_fail=True): + w.run_ninja(['-C', llvmbuilddir], ccache_stats=True) + + with w.step('check-polly'): + w.run_ninja(['-C', llvmbuilddir, 'check-polly']) + + with w.step('install-llvm', halt_on_fail=True): + w.run_ninja(['-C', llvmbuilddir, 'install'], ccache_stats=True) + + with w. step('checkout-testsuite', halt_on_fail=True): + w. checkout('https://github.com/llvm/llvm-test-suite',testsuitesrcdir) + + with w.step('configure-testsuite', halt_on_fail=True): + jobsarg = f';-j{w.jobs}' if w.jobs else '' + w.run_cmake(['cmake', + '-S', testsuitesrcdir, + '-B', testsuitebuilddir, + '-G', 'Ninja', + '-DCMAKE_BUILD_TYPE=Release', + f'-DCMAKE_C_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang', + f'-DCMAKE_CXX_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang++', + f'-DTEST_SUITE_LIT={os.path.abspath(llvmbuilddir)}/bin/llvm-lit', + f'-DTEST_SUITE_LLVM_SIZE={os.path.abspath(llvmbuilddir)}/bin/llvm-size', + "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", + "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", + '-DLLVM_LIT_ARGS=-sv{jobsarg};-o;report.json' + ]) + + with w.step('build-testsuite', halt_on_fail=True): + w. run_ninja( ['-C', testsuitebuilddir], ccache_stats=True) + + with w.step('check-testsuite'): + w.run_ninja( ['-C', testsuitebuilddir, 'check']) From b0ba18dc3305c66033b3ccb034ff4e0bf40df3bd Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Sat, 8 Nov 2025 01:10:54 +0100 Subject: [PATCH 07/10] Move directory structure --- {.buildbot/common => .ci/buildbot}/worker.py | 0 polly/{.buildbot => ci}/.gitignore | 0 polly/{.buildbot => ci}/polly-x86_64-linux-test-suite.cmake | 0 polly/{.buildbot => ci}/polly-x86_64-linux-test-suite.py | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) rename {.buildbot/common => .ci/buildbot}/worker.py (100%) rename polly/{.buildbot => ci}/.gitignore (100%) rename polly/{.buildbot => ci}/polly-x86_64-linux-test-suite.cmake (100%) rename polly/{.buildbot => ci}/polly-x86_64-linux-test-suite.py (97%) diff --git a/.buildbot/common/worker.py b/.ci/buildbot/worker.py similarity index 100% rename from .buildbot/common/worker.py rename to .ci/buildbot/worker.py diff --git a/polly/.buildbot/.gitignore b/polly/ci/.gitignore similarity index 100% rename from polly/.buildbot/.gitignore rename to polly/ci/.gitignore diff --git a/polly/.buildbot/polly-x86_64-linux-test-suite.cmake b/polly/ci/polly-x86_64-linux-test-suite.cmake similarity index 100% rename from polly/.buildbot/polly-x86_64-linux-test-suite.cmake rename to polly/ci/polly-x86_64-linux-test-suite.cmake diff --git a/polly/.buildbot/polly-x86_64-linux-test-suite.py b/polly/ci/polly-x86_64-linux-test-suite.py similarity index 97% rename from polly/.buildbot/polly-x86_64-linux-test-suite.py rename to polly/ci/polly-x86_64-linux-test-suite.py index 13424833f3a06..924c31c117189 100644 --- a/polly/.buildbot/polly-x86_64-linux-test-suite.py +++ b/polly/ci/polly-x86_64-linux-test-suite.py @@ -7,7 +7,7 @@ # Adapt to location in source tree llvmsrcroot = os.path.normpath(f"{__file__}/../../..") -sys.path.insert(0, os.path.join(llvmsrcroot, '.buildbot/common')) +sys.path.insert(0, os.path.join(llvmsrcroot, '.ci/buildbot')) import worker From 02080566fa1e1c68b4418950f8a77dc1d4a64a48 Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Sun, 9 Nov 2025 23:31:31 +0100 Subject: [PATCH 08/10] cleanup --- .ci/buildbot/worker.py | 539 +++++++++------------- polly/ci/polly-x86_64-linux-test-suite.py | 28 +- 2 files changed, 230 insertions(+), 337 deletions(-) diff --git a/.ci/buildbot/worker.py b/.ci/buildbot/worker.py index f40f0eb4d0865..33d7394b3ff16 100644 --- a/.ci/buildbot/worker.py +++ b/.ci/buildbot/worker.py @@ -1,18 +1,13 @@ import argparse import filecmp -import tempfile import os import subprocess import sys import traceback import pathlib -import subprocess -import errno -import os import re import shutil -import subprocess -import sys +import shlex from contextlib import contextmanager @@ -20,262 +15,135 @@ -def cmake_pjoin(*args): - """ - Join paths like safe_pjoin, but replace backslashes with forward - slashes on platforms where they are path separators. This prevents - CMake from choking when trying to decode what it thinks are escape - sequences in filenames. - """ - result = safe_pjoin(*args) - if os.sep == '\\': - return result.replace('\\', '/') +_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') +_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') +def _shquote_windows(txt): + """shlex.quote for Windows cmd.exe""" + txt = txt.replace('%', '%%') + quoted = re.sub(_SHQUOTE_WINDOWS_ESCAPEDCHARS, r'\\\1', txt) + if len(quoted) == len(txt) and not _SHQUOTE_WINDOWS_QUOTEDCHARS.search(txt): + return txt else: - return result - - -def report(msg): - sys.stderr.write(msg + '\n') - sys.stderr.flush() - - -def report_run_cmd(cmd, shell=False, *args, **kwargs): - """ - Print a command, then executes it using subprocess.check_call. - """ - report('Running: %s' % ((cmd if shell else shquote_cmd(cmd)),)) - sys.stderr.flush() - subprocess.check_call(cmd, shell=shell, *args, **kwargs) + return '"' + quoted + '"' -def mkdirp(path): - """Create directory path if it does not already exist.""" - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - -def rmtree(path): - """ - Remove directory path and all its subdirectories. This differs from - shutil.rmtree() in that it tries to adjust permissions so that deletion - will succeed. - """ - # Some files will not be deletable, so we set permissions that allow - # deletion before we try deleting files. - for root, dirs, files in os.walk(path): - os.chmod(root, 0o755) - for f in files: - p = os.path.join(root, f) - os.chmod(p, 0o644) - os.unlink(p) - # At this point, we should have a tree of deletable directories. - shutil.rmtree(path) - - -def safe_pjoin(dirname, *args): - """ - Join path components with os.path.join, skipping the first component - if it is None. - """ - if dirname is None: - return os.path.join(*args) +def shjoin(args): + """Convert a list of shell arguments to an appropriately quoted string.""" + if os.name in set(('nt', 'os2', 'ce')): + return ' '.join(map(_shquote_windows, args)) else: - return os.path.join(dirname, *args) + return shlex.join(args) -def _shquote_impl(txt, escaped_chars, quoted_chars): - quoted = re.sub(escaped_chars, r'\\\1', txt) - if len(quoted) == len(txt) and not quoted_chars.search(txt): - return txt - else: - return '"' + quoted + '"' -_SHQUOTE_POSIX_ESCAPEDCHARS = re.compile(r'(["`$\\])') -_SHQUOTE_POSIX_QUOTEDCHARS = re.compile('[|&;<>()\' \t\n]') -def shquote_posix(txt): - """Return txt, appropriately quoted for POSIX shells.""" - return _shquote_impl( - txt, _SHQUOTE_POSIX_ESCAPEDCHARS, _SHQUOTE_POSIX_QUOTEDCHARS) -_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') -_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') +def report(msg): + """Emit a message to the build log. Appears in red font. Lines surrounded by @@@ may be interpreted as meta-instructions.""" + print(msg,file=sys.stderr, flush=True) -def shquote_windows(txt): - """Return txt, appropriately quoted for Windows's cmd.exe.""" - return _shquote_impl( - txt.replace('%', '%%'), - _SHQUOTE_WINDOWS_ESCAPEDCHARS, _SHQUOTE_WINDOWS_QUOTEDCHARS) +def run_command(cmd, shell=False, **kwargs): + """Report which command is being run, then execute it using subprocess.check_call.""" + report(f'Running: {cmd if shell else shjoin(cmd)}') + sys.stderr.flush() + subprocess.check_call(cmd, shell=shell, **kwargs) -def shquote(txt): - """Return txt, appropriately quoted for use in a shell command.""" - if os.name in set(('nt', 'os2', 'ce')): - return shquote_windows(txt) - else: - return shquote_posix(txt) -def shquote_cmd(cmd): - """Convert a list of shell arguments to an appropriately quoted string.""" - return ' '.join(map(shquote, cmd)) -def clean_dir(path): - """ - Removes directory at path (and all its subdirectories) if it exists, - and creates an empty directory in its place. - """ - try: - rmtree(path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - mkdirp(path) +def _remove_readonly(func, path, _): + """Clear the readonly bit and reattempt the removal.""" + os.chmod(path, os.stat.S_IWRITE) + func(path) -def cmake_pjoin(*args): - """ - Join paths like safe_pjoin, but replace backslashes with forward - slashes on platforms where they are path separators. This prevents - CMake from choking when trying to decode what it thinks are escape - sequences in filenames. +def rmtree(path): """ - result = safe_pjoin(*args) - if os.sep == '\\': - return result.replace('\\', '/') - else: - return result + Remove directory path and all its subdirectories. Includes a workaround for Windows where shutil.rmtree errors on read-only files. + Taken from official Pythons docs + https://docs.python.org/3/library/shutil.html#rmtree-example + """ + shutil.rmtree(path, onexc=_remove_readonly) -def report(msg): - sys.stderr.write(msg + '\n') - sys.stderr.flush() -def report_run_cmd(cmd, shell=False, *args, **kwargs): - """ - Print a command, then executes it using subprocess.check_call. - """ - report('Running: %s' % ((cmd if shell else shquote_cmd(cmd)),)) - sys.stderr.flush() - subprocess.check_call([str(c) for c in cmd], shell=shell, *args, **kwargs) -def mkdirp(path): - """Create directory path if it does not already exist.""" - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - -def rmtree(path): - """ - Remove directory path and all its subdirectories. This differs from - shutil.rmtree() in that it tries to adjust permissions so that deletion - will succeed. - """ - # Some files will not be deletable, so we set permissions that allow - # deletion before we try deleting files. - for root, dirs, files in os.walk(path): - os.chmod(root, 0o755) - for f in files: - p = os.path.join(root, f) - os.chmod(p, 0o644) - os.unlink(p) - # At this point, we should have a tree of deletable directories. - shutil.rmtree(path) - - -def safe_pjoin(dirname, *args): - """ - Join path components with os.path.join, skipping the first component - if it is None. - """ - if dirname is None: - return os.path.join(*args) - else: - return os.path.join(dirname, *args) -def _shquote_impl(txt, escaped_chars, quoted_chars): - quoted = re.sub(escaped_chars, r'\\\1', txt) - if len(quoted) == len(txt) and not quoted_chars.search(txt): - return txt - else: - return '"' + quoted + '"' -_SHQUOTE_POSIX_ESCAPEDCHARS = re.compile(r'(["`$\\])') -_SHQUOTE_POSIX_QUOTEDCHARS = re.compile('[|&;<>()\' \t\n]') +def checkout(giturl, sourcepath): + """ + Use git to checkout the remote repository giturl at local directory sourcepath. -def shquote_posix(txt): - """Return txt, appropriately quoted for POSIX shells.""" - return _shquote_impl( - txt, _SHQUOTE_POSIX_ESCAPEDCHARS, _SHQUOTE_POSIX_QUOTEDCHARS) + If the repository already exists, clear all local changes and check out the latest main branch. + """ + if not os.path.exists(sourcepath): + run_command(['git', 'clone', giturl, sourcepath]) + # Reset repository state no matter what there was before + run_command(['git', '-C', sourcepath, 'stash', '--all']) + run_command(['git', '-C', sourcepath, 'stash', 'clear']) -_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') -_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') + # Fetch and checkout the newest + run_command(['git', '-C', sourcepath, 'fetch', 'origin']) + run_command(['git', '-C', sourcepath, 'checkout', 'origin/main', '--detach']) -def shquote_windows(txt): - """Return txt, appropriately quoted for Windows's cmd.exe.""" - return _shquote_impl( - txt.replace('%', '%%'), - _SHQUOTE_WINDOWS_ESCAPEDCHARS, _SHQUOTE_WINDOWS_QUOTEDCHARS) -def shquote(txt): - """Return txt, appropriately quoted for use in a shell command.""" - if os.name in set(('nt', 'os2', 'ce')): - return shquote_windows(txt) - else: - return shquote_posix(txt) -def shquote_cmd(cmd): - """Convert a list of shell arguments to an appropriately quoted string.""" - return ' '.join(map(shquote, cmd)) -def relative_if_possible(path, relative_to): - path = os.path.normpath(path) - try: - result = os.path.relpath(path, start=relative_to) - return result if result else path - except Exception: - return path +@contextmanager +def step(step_name, halt_on_fail=False): + """Report a new build step being started. + Use like this:: + with step("greet-step"): + report("Hello World!") + """ + # Barrier to separate stdio output for the the previous step + sys.stderr.flush() + sys.stdout.flush() + report(f'@@@BUILD_STEP {step_name}@@@') + if halt_on_fail: + report('@@@HALT_ON_FAILURE@@@') + try: + yield + except Exception as e: + if isinstance(e, subprocess.CalledProcessError): + report(f'{shjoin(e.cmd)} exited with return code {e.returncode}.' ) + report('@@@STEP_FAILURE@@@') + else: + traceback.print_exc() + report('@@@STEP_EXCEPTION@@@') + if halt_on_fail: + # Do not continue with the next steps, but allow except/finally blocks to execute + raise e -def first_true(*args): - for a in args: - if a: - return a - return None -def first_nonnull(*args): - for a in args: - if a: - return a - return None class Worker: + """Helper class to keep context in a worker.run() environment""" + def __init__(self,args,clean,clobber,workdir,jobs,cachefile,llvmsrcroot): self.args=args self.clean =clean @@ -286,62 +154,154 @@ def __init__(self,args,clean,clobber,workdir,jobs,cachefile,llvmsrcroot): self.llvmsrcroot=llvmsrcroot def in_llvmsrc(self, path): + """Convert a path in the llvm-project source checkout to an absolute path""" return os.path.join(self.llvmsrcroot, path) def in_workdir(self, path): + """Convert a path in the workdir to an absolute path""" return os.path.join(self.workdir, path) + + def run_ninja( self, targets :list=[], *, builddir, ccache_stats:bool=False, **kwargs): + """Run ninja in builddir. If self.jobs is set, automatically adds an -j option to set the number of parallel jobs. + + Parameters + ---------- + targets : list + List of build targets; build the default target 'all' if list is empty + builddir + Directory of the build.ninja file + ccache_stats : bool + If true, also emit ccache statistics when finishing the build + """ + cmd = ['ninja'] + if builddir is not None: + cmd += ['-C', builddir] + cmd += targets + if self.jobs: + cmd .append(f'-j{self.jobs}') + if ccache_stats: + run_command(['ccache', '-z']) + try: + run_command(cmd, **kwargs) + finally: + # TODO: Pipe to stderr to separate from build log itself + run_command(['ccache', '-sv']) + else: + run_command(cmd, **kwargs) + @contextmanager def step(self, step_name, halt_on_fail=False): + """Convenience wrapper for step()""" with step(step_name,halt_on_fail=halt_on_fail) as s: yield s - def run_cmake(self, cmakeargs): - report_run_cmd(['cmake', *cmakeargs]) + def run_command(self, *args, **kwargs): + """Convenience wrapper for run_command()""" + return run_command(*args, **kwargs) - def run_ninja(args, targets, ccache_stats=False, **kwargs): - cmd = ['ninja', *targets] - if args.jobs: - args .append(f'-j{args.jobs}') - if ccache_stats: - report_run_cmd(['ccache', '-z']) - report_run_cmd(cmd, **kwargs) - if ccache_stats: - report_run_cmd(['ccache', '-sv']) + + def rmtree (self, *args, **kwargs): + """Convenience wrapper for rmtree()""" + return rmtree(*args,*kwargs) def checkout (self,giturl, sourcepath): - return checkout(giturl, sourcepath) + """Convenience wrapper for checkout()""" + return checkout(giturl, sourcepath) - -@contextmanager -def run(scriptname, llvmsrcroot, parser=None ,clobberpaths=[], workerjobs=None): - for k,v in os.environ.items(): - print(f"{k}={v}") - - os.environ['NINJA_STATUS'] = "[%p/%es :: %u->%r->%f (of %t)] " +def convert_bool(v): + """Convert input to bool type - jobs_default = first_true( os.environ.get('BUILDBOT_JOBS') , workerjobs ) + Use to convert the value of bool environment variables. Specifically, the buildbot master sets 'false' to build properties, which by default Python would interpret as true-ish. + """ + match v: + case None: + return False + case bool(b): + return b + case str(s): + return not s.strip().upper() in ["", "0", "N", "NO", "FALSE", "OFF"] + case _: + return bool(v) - stem = pathlib.Path(scriptname).stem +def relative_if_possible(path, relative_to): + """Like os.path.relpath, but does not fail if path is not a parent of relative_to; keeps the original path in that case""" + path = os.path.normpath(path) + if not os.path.isabs(path): + # Path is already relative (assumed to relative_to) + return path + try: + result = os.path.relpath(path, start=relative_to) + return result if result else path + except ValueError: + return path - parser = parser or argparse.ArgumentParser(allow_abbrev=True, description="Run the buildbot builder configuration; when executed without arguments, builds the llvm-project checkout this file is in, using cwd to write all files") - parser.add_argument('--jobs', '-j', default=jobs_default, help='Override the number of default jobs') - parser.add_argument('--clean', type=bool, default=os.environ.get('BUILDBOT_CLEAN') , help='Whether to delete source-, build-, and install-dirs (i.e. remove any files leftover from a previous run, if any) before running') - parser.add_argument('--clobber', type=bool, default=os.environ.get('BUILDBOT_CLOBBER') or os.environ.get('BUILDBOT_CLEAN_OBJ'), help='Whether to delete build- and install-dirs before running') - parser.add_argument('--workdir', default=f'{stem}.workdir', help="Use this dir as workdir") - parser.add_argument('--cachefile', default=relative_if_possible(pathlib.Path(scriptname).with_suffix('.cmake'), llvmsrcroot), help='CMake cache seed') +@contextmanager +def run(scriptpath, llvmsrcroot, parser=None ,clobberpaths=[], workerjobs=None, always_clobber=False): + """Runs the boilerplate for a ScriptedBuilder buildbot. It is not necessary to use this function (one can also all run_command() etc. directly), but allows for some more flexibility and safety checks. Arguments passed to this function represent the worker configuration. + + We use the term 'clean' for resetting the worker to an empty state. This involves deleting ${prefix}/llvm.src as well as ${prefix}/build. + The term 'clobber' means deleting build artifacts, but not already downloaded git repositories. Build artifacts including build- and install-directories, but not source directories. Changes in the llvm.src directory will be reset before the next build anyway. Clobber is necessary if the build instructions change. Otherwise, we try an incremental build. We consider 'clean' to imply 'clean_obj'. + + A buildbot worker will invoke this script using this directory structure, where ${prefix} is a dedicated directory for this builder: + ${prefix}/llvm.src # Checkout location for the llvm-source + ${prefix}/build # cwd when launching the build script + + The build script is called with --workdir=. parameter, i.e. the build artifacts are written into ${prefix}/build. When cleaning, the worker (NOT the build script) will delete ${prefix}/llvm.src; Deleting any contents of ${prefix}/build is to be done by the builder script, e.g. by this function. The builder script can choose to not delete the complete workdir, e.g. additional source checkouts such as the llvm-test-suite. + + The buildbot master will set the 'clean' build property and the environment variable BUILDBOT_CLEAN when in the GUI the option "Clean source code and build directory" is checked by the user. The 'clean_obj' build property and the BUILDBOT_CLEAN_OBJ environment variable will be set when either the "Clean build directory" GUI option is set, or the master detects a change to a CMakeLists.txt or *.cmake file. + + Parameters + ---------- + scriptpath + Pass __file__ from the main builder script. + llvmsrcroot + Absolute path to the llvm-project source checkout. Since the builder script is supposed to be a part of llvm-project itself, the builder script can compute it from __file__. + parser + Use this argparse.ArgumentParser instead of creating a new one. Allows adding additional command line switched in addition to the pre-defined ones. Build script are encouraged to apply the pre-defined switches. + clobberpaths + Directories relative to workdir that need to be deleted if the build configuration changes (due to changes of CMakeLists.txt or changes of configuration parameters). Typically, only source checkouts are not deleted. + workerjobs + Default number of build and test jobs; If set, expected to be the number of jobs of the actual buildbot worker that executes this script. Can be overridden using the --jobs parameter so in case someone needs to reproduce this build, they can adjust the number of jobs for the reproducer platform. Alternatively, the worker can set the BUILDBOT_JOBS environment variable or keep ninja/llvm-lit defaults. + always_clobber + Always clobber the build artifacts, i.e. disable incremental builds. +""" + + assert os.path.isabs(scriptpath) + assert os.path.isabs(llvmsrcroot) + + + jobs_default=None + if jobs_env := os.environ.get('BUILDBOT_JOBS'): + jobs_default = int(jobs_env) + if not jobs_default: + jobs_default = workerjobs + if not jobs_default: + jobs_default=None + + + stem = pathlib.Path(scriptpath).stem + workdir_default = f'{stem}.workdir' + + parser = parser or argparse.ArgumentParser(allow_abbrev=True, description=f"When executed without arguments, builds the worker's LLVM build configuration in {os.path.abspath(workdir_default)}. Some build configuration parameters can be altered using the following switches:") + parser.add_argument('--workdir', default=workdir_default, help="Use this dir as workdir to write the build artifact into. --workdir=. uses the current directory.\nWarning: This directory might be deleted") + parser.add_argument('--cachefile', default=relative_if_possible(pathlib.Path(scriptpath).with_suffix('.cmake'), llvmsrcroot), help='File containing the initial values for the CMakeCache.txt for the llvm build.') + parser.add_argument('--clean', type=bool, default=convert_bool(os.environ.get('BUILDBOT_CLEAN')), help='Delete the entire workdir before starting the build, including source directories') + parser.add_argument('--clobber', type=bool, default=always_clobber or convert_bool(os.environ.get('BUILDBOT_CLOBBER')) or convert_bool(os.environ.get('BUILDBOT_CLEAN_OBJ')), help='Delete build artifacts before starting the build') + parser.add_argument('--jobs', '-j', default=jobs_default, help='Number of build- and test-jobs') args = parser.parse_args() - print("cwd:",os.getcwd()) + workdir = args.workdir + clean =args.clean clobber = args.clobber cachefile = os.path.join(llvmsrcroot, args.cachefile) oldcwd = os.getcwd() @@ -352,18 +312,34 @@ def run(scriptname, llvmsrcroot, parser=None ,clobberpaths=[], workerjobs=None): # Force clobber if cache file has changed; a new cachefile does not override entries already present in CMakeCache.txt if not filecmp.cmp( os.path.join(llvmsrcroot, args.cachefile), prevcachepath, shallow=False): clobber = True - #shutil.copyfile(cachefile) - w = Worker(args, clean=args.clean , clobber=clobber, workdir=workdir, jobs=args.jobs, cachefile=cachefile, llvmsrcroot=llvmsrcroot) - #if workdir: - if args.clean: + w = Worker(args, clean=clean , clobber=clobber, workdir=workdir, jobs=args.jobs, cachefile=cachefile, llvmsrcroot=llvmsrcroot) + + + + # Safety check + parentdir = os.path.dirname(scriptpath) + while True: + if os.path.samefile( parentdir, workdir) : + raise Exception(f"Cannot use {workdir} as workdir; a '--clean' build would rmtree the llvm-project source as well") + newparentdir = os.path.dirname(parentdir) + if newparentdir ==parentdir: + break + parentdir = newparentdir + + + + if clean: + # Ensure that the cwd is not the directory we are going to delete. This would not work under Windows. We will chdir to workdir later anyway. + os.chdir('/') + with w.step(f'clean'): - clean_dir(workdir) + rmtree(workdir) elif clobber: with w.step(f'clobber'): for d in clobberpaths: - clean_dir(os.path.join(workdir, d)) + rmtree(os.path.join(workdir, d)) os.path.unlink(prevcachepath) os.makedirs(workdir, exist_ok=True) @@ -371,90 +347,9 @@ def run(scriptname, llvmsrcroot, parser=None ,clobberpaths=[], workerjobs=None): # Remember used cachefile in case it changes if cachefile: - shutil.copy2(os.path.join(oldcwd,llvmsrcroot, args.cachefile), os.path.join(oldcwd, prevcachepath )) + shutil.copy(os.path.join(oldcwd,llvmsrcroot, args.cachefile), os.path.join(oldcwd, prevcachepath )) + os.environ['NINJA_STATUS'] = "[%p/%es :: %u->%r->%f (of %t)] " yield w - #else: - # with tempfile.TemporaryDirectory(prefix = stem) as tmpdir: - # os.chdir(tmpdir) - # yield Worker(args) - - - - - - -@contextmanager -def step(step_name, halt_on_fail=False): - sys.stderr.flush() - sys.stdout.flush() - report(f'@@@BUILD_STEP {step_name}@@@') - if halt_on_fail: - report('@@@HALT_ON_FAILURE@@@') - try: - yield - except Exception as e: - if isinstance(e, subprocess.CalledProcessError): - report(f'{e.cmd} exited with return code {e.returncode}.' ) - report('@@@STEP_FAILURE@@@') - else: - traceback.print_exc() - report('@@@STEP_EXCEPTION@@@') - if halt_on_fail: - exit(1) - - -def get_steps(makefile): - try: - make_cmd = build_make_cmd(makefile, 'get-steps') - raw_steps = capture_cmd_stdout(make_cmd) - return raw_steps.decode('utf-8').split('\n')[:-1] - except: - return [] - - -def build_make_cmd(makefile, target, make_vars={}): - make_cmd = ['make', '-f', makefile] - if not target is None: - make_cmd.append(target) - for k,v in make_vars.items(): - make_cmd += ["{}={}".format(k, v)] - return make_cmd - - -def capture_cmd_stdout(cmd, **kwargs): - return subprocess.run(cmd, shell=False, check=True, stdout=subprocess.PIPE, **kwargs).stdout - - -def run_command(cmd, **kwargs): - report_run_cmd(cmd, **kwargs) - - - - - -def checkout(giturl, sourcepath): - if not os.path.exists(sourcepath): - run_command(['git', 'clone', giturl, sourcepath]) - - # Reset repository state no matter what there was before - run_command(['git', '-C', sourcepath, 'stash', '--all']) - run_command(['git', '-C', sourcepath, 'stash', 'clear']) - - # Fetch and checkout the newest - run_command(['git', '-C', sourcepath, 'fetch', 'origin']) - run_command(['git', '-C', sourcepath, 'checkout', 'origin/main', '--detach']) - - -def clean_on_request(args, always=[],on_clobber=[],on_clean=[]): - cleanset = always - if args.clobber or args.clean: - # Clean implies clobber - cleanset += on_clobber - if args.clean: - cleanset += on_clean - for d in cleanset: - with step(f'delete-{os.path.basename(d)}'): - clean_dir(d) diff --git a/polly/ci/polly-x86_64-linux-test-suite.py b/polly/ci/polly-x86_64-linux-test-suite.py index 924c31c117189..b409c4019e285 100644 --- a/polly/ci/polly-x86_64-linux-test-suite.py +++ b/polly/ci/polly-x86_64-linux-test-suite.py @@ -1,8 +1,7 @@ -#! /usr/bin/python3 +#! /usr/bin/env python3 import os import sys -import pathlib # Adapt to location in source tree llvmsrcroot = os.path.normpath(f"{__file__}/../../..") @@ -11,15 +10,14 @@ import worker - llvmbuilddir = "llvm.build" llvminstalldir = 'llvm.install' testsuitesrcdir = "testsuite.src" testsuitebuilddir = "testsuite.build" -with worker.run(scriptname=__file__,llvmsrcroot=llvmsrcroot,clobberpaths = [llvmbuilddir,testsuitebuilddir, llvminstalldir]) as w: +with worker.run(__file__,llvmsrcroot,clobberpaths = [llvmbuilddir,testsuitebuilddir, llvminstalldir]) as w: with w.step('configure-llvm', halt_on_fail=True): - cmakeargs =[ + cmakecmd =['cmake', '-S', w.in_llvmsrc( 'llvm'), '-B', llvmbuilddir, '-G', 'Ninja', @@ -27,24 +25,24 @@ f'-DCMAKE_INSTALL_PREFIX={llvminstalldir}' ] if w.jobs: - cmakeargs.append( f'-DLLVM_LIT_ARGS=-sv;-j{w.jobs}') - w.run_cmake(cmakeargs) + cmakecmd.append( f'-DLLVM_LIT_ARGS=-sv;-j{w.jobs}') + w.run_command(cmakecmd) with w.step('build-llvm', halt_on_fail=True): - w.run_ninja(['-C', llvmbuilddir], ccache_stats=True) + w.run_ninja(builddir = llvmbuilddir, ccache_stats=True) with w.step('check-polly'): - w.run_ninja(['-C', llvmbuilddir, 'check-polly']) + w.run_ninja(['check-polly'], builddir= llvmbuilddir,targets= []) with w.step('install-llvm', halt_on_fail=True): - w.run_ninja(['-C', llvmbuilddir, 'install'], ccache_stats=True) + w.run_ninja(['install'], builddir= llvmbuilddir) with w. step('checkout-testsuite', halt_on_fail=True): w. checkout('https://github.com/llvm/llvm-test-suite',testsuitesrcdir) with w.step('configure-testsuite', halt_on_fail=True): jobsarg = f';-j{w.jobs}' if w.jobs else '' - w.run_cmake(['cmake', + w.run_command(['cmake', '-S', testsuitesrcdir, '-B', testsuitebuilddir, '-G', 'Ninja', @@ -55,12 +53,12 @@ f'-DTEST_SUITE_LLVM_SIZE={os.path.abspath(llvmbuilddir)}/bin/llvm-size', "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", - '-DLLVM_LIT_ARGS=-sv{jobsarg};-o;report.json' - ]) + '-DLLVM_LIT_ARGS=-sv{jobsarg};-o;report.json'] + ) with w.step('build-testsuite', halt_on_fail=True): - w. run_ninja( ['-C', testsuitebuilddir], ccache_stats=True) + w. run_ninja( builddir=testsuitebuilddir) with w.step('check-testsuite'): - w.run_ninja( ['-C', testsuitebuilddir, 'check']) + w.run_ninja( ['check'], builddir=testsuitebuilddir) From 05e0d30cbbe8802c2b72997791a4ca7a96abe087 Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Sun, 9 Nov 2025 23:38:25 +0100 Subject: [PATCH 09/10] darker --- .ci/buildbot/worker.py | 289 +++++++++++----------- polly/ci/polly-x86_64-linux-test-suite.py | 84 ++++--- 2 files changed, 189 insertions(+), 184 deletions(-) diff --git a/.ci/buildbot/worker.py b/.ci/buildbot/worker.py index 33d7394b3ff16..0c1c538babc6f 100644 --- a/.ci/buildbot/worker.py +++ b/.ci/buildbot/worker.py @@ -1,68 +1,53 @@ import argparse import filecmp import os -import subprocess -import sys -import traceback import pathlib import re -import shutil import shlex +import shutil +import subprocess +import sys +import traceback from contextlib import contextmanager +_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') +_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile("[ \t\n]") - - - -_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') -_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile('[ \t\n]') def _shquote_windows(txt): """shlex.quote for Windows cmd.exe""" - txt = txt.replace('%', '%%') - quoted = re.sub(_SHQUOTE_WINDOWS_ESCAPEDCHARS, r'\\\1', txt) + txt = txt.replace("%", "%%") + quoted = re.sub(_SHQUOTE_WINDOWS_ESCAPEDCHARS, r"\\\1", txt) if len(quoted) == len(txt) and not _SHQUOTE_WINDOWS_QUOTEDCHARS.search(txt): return txt else: return '"' + quoted + '"' - def shjoin(args): """Convert a list of shell arguments to an appropriately quoted string.""" - if os.name in set(('nt', 'os2', 'ce')): - return ' '.join(map(_shquote_windows, args)) + if os.name in set(("nt", "os2", "ce")): + return " ".join(map(_shquote_windows, args)) else: - return shlex.join(args) - - - - - - + return shlex.join(args) def report(msg): """Emit a message to the build log. Appears in red font. Lines surrounded by @@@ may be interpreted as meta-instructions.""" - print(msg,file=sys.stderr, flush=True) + print(msg, file=sys.stderr, flush=True) - -def run_command(cmd, shell=False, **kwargs): +def run_command(cmd, shell=False, **kwargs): """Report which command is being run, then execute it using subprocess.check_call.""" - report(f'Running: {cmd if shell else shjoin(cmd)}') + report(f"Running: {cmd if shell else shjoin(cmd)}") sys.stderr.flush() - subprocess.check_call(cmd, shell=shell, **kwargs) - - - - + subprocess.check_call(cmd, shell=shell, **kwargs) def _remove_readonly(func, path, _): - """Clear the readonly bit and reattempt the removal.""" - os.chmod(path, os.stat.S_IWRITE) - func(path) + """Clear the readonly bit and reattempt the removal.""" + os.chmod(path, os.stat.S_IWRITE) + func(path) def rmtree(path): @@ -75,15 +60,6 @@ def rmtree(path): shutil.rmtree(path, onexc=_remove_readonly) - - - - - - - - - def checkout(giturl, sourcepath): """ Use git to checkout the remote repository giturl at local directory sourcepath. @@ -91,22 +67,15 @@ def checkout(giturl, sourcepath): If the repository already exists, clear all local changes and check out the latest main branch. """ if not os.path.exists(sourcepath): - run_command(['git', 'clone', giturl, sourcepath]) + run_command(["git", "clone", giturl, sourcepath]) # Reset repository state no matter what there was before - run_command(['git', '-C', sourcepath, 'stash', '--all']) - run_command(['git', '-C', sourcepath, 'stash', 'clear']) + run_command(["git", "-C", sourcepath, "stash", "--all"]) + run_command(["git", "-C", sourcepath, "stash", "clear"]) # Fetch and checkout the newest - run_command(['git', '-C', sourcepath, 'fetch', 'origin']) - run_command(['git', '-C', sourcepath, 'checkout', 'origin/main', '--detach']) - - - - - - - + run_command(["git", "-C", sourcepath, "fetch", "origin"]) + run_command(["git", "-C", sourcepath, "checkout", "origin/main", "--detach"]) @contextmanager @@ -121,37 +90,34 @@ def step(step_name, halt_on_fail=False): sys.stderr.flush() sys.stdout.flush() - report(f'@@@BUILD_STEP {step_name}@@@') + report(f"@@@BUILD_STEP {step_name}@@@") if halt_on_fail: - report('@@@HALT_ON_FAILURE@@@') + report("@@@HALT_ON_FAILURE@@@") try: yield except Exception as e: if isinstance(e, subprocess.CalledProcessError): - report(f'{shjoin(e.cmd)} exited with return code {e.returncode}.' ) - report('@@@STEP_FAILURE@@@') + report(f"{shjoin(e.cmd)} exited with return code {e.returncode}.") + report("@@@STEP_FAILURE@@@") else: traceback.print_exc() - report('@@@STEP_EXCEPTION@@@') + report("@@@STEP_EXCEPTION@@@") if halt_on_fail: # Do not continue with the next steps, but allow except/finally blocks to execute raise e - - - class Worker: """Helper class to keep context in a worker.run() environment""" - def __init__(self,args,clean,clobber,workdir,jobs,cachefile,llvmsrcroot): - self.args=args - self.clean =clean - self.clobber=clobber - self.workdir=workdir - self.jobs=jobs - self.cachefile =cachefile - self.llvmsrcroot=llvmsrcroot + def __init__(self, args, clean, clobber, workdir, jobs, cachefile, llvmsrcroot): + self.args = args + self.clean = clean + self.clobber = clobber + self.workdir = workdir + self.jobs = jobs + self.cachefile = cachefile + self.llvmsrcroot = llvmsrcroot def in_llvmsrc(self, path): """Convert a path in the llvm-project source checkout to an absolute path""" @@ -161,8 +127,9 @@ def in_workdir(self, path): """Convert a path in the workdir to an absolute path""" return os.path.join(self.workdir, path) - - def run_ninja( self, targets :list=[], *, builddir, ccache_stats:bool=False, **kwargs): + def run_ninja( + self, targets: list = [], *, builddir, ccache_stats: bool = False, **kwargs + ): """Run ninja in builddir. If self.jobs is set, automatically adds an -j option to set the number of parallel jobs. Parameters @@ -174,43 +141,39 @@ def run_ninja( self, targets :list=[], *, builddir, ccache_stats:bool=False, **k ccache_stats : bool If true, also emit ccache statistics when finishing the build """ - cmd = ['ninja'] + cmd = ["ninja"] if builddir is not None: - cmd += ['-C', builddir] + cmd += ["-C", builddir] cmd += targets if self.jobs: - cmd .append(f'-j{self.jobs}') + cmd.append(f"-j{self.jobs}") if ccache_stats: - run_command(['ccache', '-z']) + run_command(["ccache", "-z"]) try: run_command(cmd, **kwargs) finally: - # TODO: Pipe to stderr to separate from build log itself - run_command(['ccache', '-sv']) + # TODO: Pipe to stderr to separate from build log itself + run_command(["ccache", "-sv"]) else: - run_command(cmd, **kwargs) + run_command(cmd, **kwargs) @contextmanager def step(self, step_name, halt_on_fail=False): """Convenience wrapper for step()""" - with step(step_name,halt_on_fail=halt_on_fail) as s: + with step(step_name, halt_on_fail=halt_on_fail) as s: yield s def run_command(self, *args, **kwargs): - """Convenience wrapper for run_command()""" - return run_command(*args, **kwargs) - - - def rmtree (self, *args, **kwargs): - """Convenience wrapper for rmtree()""" - return rmtree(*args,*kwargs) - - def checkout (self,giturl, sourcepath): - """Convenience wrapper for checkout()""" - return checkout(giturl, sourcepath) - + """Convenience wrapper for run_command()""" + return run_command(*args, **kwargs) + def rmtree(self, *args, **kwargs): + """Convenience wrapper for rmtree()""" + return rmtree(*args, *kwargs) + def checkout(self, giturl, sourcepath): + """Convenience wrapper for checkout()""" + return checkout(giturl, sourcepath) def convert_bool(v): @@ -229,22 +192,28 @@ def convert_bool(v): return bool(v) - def relative_if_possible(path, relative_to): - """Like os.path.relpath, but does not fail if path is not a parent of relative_to; keeps the original path in that case""" - path = os.path.normpath(path) - if not os.path.isabs(path): - # Path is already relative (assumed to relative_to) - return path - try: - result = os.path.relpath(path, start=relative_to) - return result if result else path - except ValueError: - return path + """Like os.path.relpath, but does not fail if path is not a parent of relative_to; keeps the original path in that case""" + path = os.path.normpath(path) + if not os.path.isabs(path): + # Path is already relative (assumed to relative_to) + return path + try: + result = os.path.relpath(path, start=relative_to) + return result if result else path + except ValueError: + return path @contextmanager -def run(scriptpath, llvmsrcroot, parser=None ,clobberpaths=[], workerjobs=None, always_clobber=False): +def run( + scriptpath, + llvmsrcroot, + parser=None, + clobberpaths=[], + workerjobs=None, + always_clobber=False, +): """Runs the boilerplate for a ScriptedBuilder buildbot. It is not necessary to use this function (one can also all run_command() etc. directly), but allows for some more flexibility and safety checks. Arguments passed to this function represent the worker configuration. We use the term 'clean' for resetting the worker to an empty state. This involves deleting ${prefix}/llvm.src as well as ${prefix}/build. @@ -272,74 +241,103 @@ def run(scriptpath, llvmsrcroot, parser=None ,clobberpaths=[], workerjobs=None, Default number of build and test jobs; If set, expected to be the number of jobs of the actual buildbot worker that executes this script. Can be overridden using the --jobs parameter so in case someone needs to reproduce this build, they can adjust the number of jobs for the reproducer platform. Alternatively, the worker can set the BUILDBOT_JOBS environment variable or keep ninja/llvm-lit defaults. always_clobber Always clobber the build artifacts, i.e. disable incremental builds. -""" + """ assert os.path.isabs(scriptpath) assert os.path.isabs(llvmsrcroot) - - jobs_default=None - if jobs_env := os.environ.get('BUILDBOT_JOBS'): + jobs_default = None + if jobs_env := os.environ.get("BUILDBOT_JOBS"): jobs_default = int(jobs_env) if not jobs_default: jobs_default = workerjobs if not jobs_default: - jobs_default=None - + jobs_default = None stem = pathlib.Path(scriptpath).stem - workdir_default = f'{stem}.workdir' - - parser = parser or argparse.ArgumentParser(allow_abbrev=True, description=f"When executed without arguments, builds the worker's LLVM build configuration in {os.path.abspath(workdir_default)}. Some build configuration parameters can be altered using the following switches:") - parser.add_argument('--workdir', default=workdir_default, help="Use this dir as workdir to write the build artifact into. --workdir=. uses the current directory.\nWarning: This directory might be deleted") - parser.add_argument('--cachefile', default=relative_if_possible(pathlib.Path(scriptpath).with_suffix('.cmake'), llvmsrcroot), help='File containing the initial values for the CMakeCache.txt for the llvm build.') - parser.add_argument('--clean', type=bool, default=convert_bool(os.environ.get('BUILDBOT_CLEAN')), help='Delete the entire workdir before starting the build, including source directories') - parser.add_argument('--clobber', type=bool, default=always_clobber or convert_bool(os.environ.get('BUILDBOT_CLOBBER')) or convert_bool(os.environ.get('BUILDBOT_CLEAN_OBJ')), help='Delete build artifacts before starting the build') - parser.add_argument('--jobs', '-j', default=jobs_default, help='Number of build- and test-jobs') + workdir_default = f"{stem}.workdir" + + parser = parser or argparse.ArgumentParser( + allow_abbrev=True, + description=f"When executed without arguments, builds the worker's LLVM build configuration in {os.path.abspath(workdir_default)}. Some build configuration parameters can be altered using the following switches:", + ) + parser.add_argument( + "--workdir", + default=workdir_default, + help="Use this dir as workdir to write the build artifact into. --workdir=. uses the current directory.\nWarning: This directory might be deleted", + ) + parser.add_argument( + "--cachefile", + default=relative_if_possible( + pathlib.Path(scriptpath).with_suffix(".cmake"), llvmsrcroot + ), + help="File containing the initial values for the CMakeCache.txt for the llvm build.", + ) + parser.add_argument( + "--clean", + type=bool, + default=convert_bool(os.environ.get("BUILDBOT_CLEAN")), + help="Delete the entire workdir before starting the build, including source directories", + ) + parser.add_argument( + "--clobber", + type=bool, + default=always_clobber + or convert_bool(os.environ.get("BUILDBOT_CLOBBER")) + or convert_bool(os.environ.get("BUILDBOT_CLEAN_OBJ")), + help="Delete build artifacts before starting the build", + ) + parser.add_argument( + "--jobs", "-j", default=jobs_default, help="Number of build- and test-jobs" + ) args = parser.parse_args() - - - workdir = args.workdir - clean =args.clean + workdir = args.workdir + clean = args.clean clobber = args.clobber cachefile = os.path.join(llvmsrcroot, args.cachefile) oldcwd = os.getcwd() - - prevcachepath = os.path.join(workdir, 'prevcache.cmake') + prevcachepath = os.path.join(workdir, "prevcache.cmake") if cachefile and os.path.exists(prevcachepath): # Force clobber if cache file has changed; a new cachefile does not override entries already present in CMakeCache.txt - if not filecmp.cmp( os.path.join(llvmsrcroot, args.cachefile), prevcachepath, shallow=False): + if not filecmp.cmp( + os.path.join(llvmsrcroot, args.cachefile), prevcachepath, shallow=False + ): clobber = True - - w = Worker(args, clean=clean , clobber=clobber, workdir=workdir, jobs=args.jobs, cachefile=cachefile, llvmsrcroot=llvmsrcroot) - - + w = Worker( + args, + clean=clean, + clobber=clobber, + workdir=workdir, + jobs=args.jobs, + cachefile=cachefile, + llvmsrcroot=llvmsrcroot, + ) # Safety check parentdir = os.path.dirname(scriptpath) while True: - if os.path.samefile( parentdir, workdir) : - raise Exception(f"Cannot use {workdir} as workdir; a '--clean' build would rmtree the llvm-project source as well") + if os.path.samefile(parentdir, workdir): + raise Exception( + f"Cannot use {workdir} as workdir; a '--clean' build would rmtree the llvm-project source as well" + ) newparentdir = os.path.dirname(parentdir) - if newparentdir ==parentdir: - break + if newparentdir == parentdir: + break parentdir = newparentdir - - if clean: - # Ensure that the cwd is not the directory we are going to delete. This would not work under Windows. We will chdir to workdir later anyway. - os.chdir('/') + # Ensure that the cwd is not the directory we are going to delete. This would not work under Windows. We will chdir to workdir later anyway. + os.chdir("/") - with w.step(f'clean'): - rmtree(workdir) + with w.step(f"clean"): + rmtree(workdir) elif clobber: - with w.step(f'clobber'): + with w.step(f"clobber"): for d in clobberpaths: - rmtree(os.path.join(workdir, d)) + rmtree(os.path.join(workdir, d)) os.path.unlink(prevcachepath) os.makedirs(workdir, exist_ok=True) @@ -347,9 +345,10 @@ def run(scriptpath, llvmsrcroot, parser=None ,clobberpaths=[], workerjobs=None, # Remember used cachefile in case it changes if cachefile: - shutil.copy(os.path.join(oldcwd,llvmsrcroot, args.cachefile), os.path.join(oldcwd, prevcachepath )) + shutil.copy( + os.path.join(oldcwd, llvmsrcroot, args.cachefile), + os.path.join(oldcwd, prevcachepath), + ) - os.environ['NINJA_STATUS'] = "[%p/%es :: %u->%r->%f (of %t)] " + os.environ["NINJA_STATUS"] = "[%p/%es :: %u->%r->%f (of %t)] " yield w - - diff --git a/polly/ci/polly-x86_64-linux-test-suite.py b/polly/ci/polly-x86_64-linux-test-suite.py index b409c4019e285..e50c39e9757b0 100644 --- a/polly/ci/polly-x86_64-linux-test-suite.py +++ b/polly/ci/polly-x86_64-linux-test-suite.py @@ -6,59 +6,65 @@ # Adapt to location in source tree llvmsrcroot = os.path.normpath(f"{__file__}/../../..") -sys.path.insert(0, os.path.join(llvmsrcroot, '.ci/buildbot')) +sys.path.insert(0, os.path.join(llvmsrcroot, ".ci/buildbot")) import worker - llvmbuilddir = "llvm.build" -llvminstalldir = 'llvm.install' +llvminstalldir = "llvm.install" testsuitesrcdir = "testsuite.src" testsuitebuilddir = "testsuite.build" -with worker.run(__file__,llvmsrcroot,clobberpaths = [llvmbuilddir,testsuitebuilddir, llvminstalldir]) as w: - with w.step('configure-llvm', halt_on_fail=True): - cmakecmd =['cmake', - '-S', w.in_llvmsrc( 'llvm'), - '-B', llvmbuilddir, - '-G', 'Ninja', - '-C', w.in_llvmsrc( w.cachefile), - f'-DCMAKE_INSTALL_PREFIX={llvminstalldir}' +with worker.run( + __file__, + llvmsrcroot, + clobberpaths=[llvmbuilddir, testsuitebuilddir, llvminstalldir], +) as w: + with w.step("configure-llvm", halt_on_fail=True): + cmakecmd = [ + "cmake", + f'-S{w.in_llvmsrc("llvm")}', + "-B{llvmbuilddir}", + "-GNinja", + f"-C{w.in_llvmsrc(w.cachefile)}", + f"-DCMAKE_INSTALL_PREFIX={llvminstalldir}", ] if w.jobs: - cmakecmd.append( f'-DLLVM_LIT_ARGS=-sv;-j{w.jobs}') + cmakecmd.append(f"-DLLVM_LIT_ARGS=-sv;-j{w.jobs}") w.run_command(cmakecmd) - with w.step('build-llvm', halt_on_fail=True): - w.run_ninja(builddir = llvmbuilddir, ccache_stats=True) + with w.step("build-llvm", halt_on_fail=True): + w.run_ninja(builddir=llvmbuilddir, ccache_stats=True) - with w.step('check-polly'): - w.run_ninja(['check-polly'], builddir= llvmbuilddir,targets= []) + with w.step("check-polly"): + w.run_ninja(["check-polly"], builddir=llvmbuilddir, targets=[]) - with w.step('install-llvm', halt_on_fail=True): - w.run_ninja(['install'], builddir= llvmbuilddir) + with w.step("install-llvm", halt_on_fail=True): + w.run_ninja(["install"], builddir=llvmbuilddir) - with w. step('checkout-testsuite', halt_on_fail=True): - w. checkout('https://github.com/llvm/llvm-test-suite',testsuitesrcdir) + with w.step("checkout-testsuite", halt_on_fail=True): + w.checkout("https://github.com/llvm/llvm-test-suite", testsuitesrcdir) - with w.step('configure-testsuite', halt_on_fail=True): - jobsarg = f';-j{w.jobs}' if w.jobs else '' - w.run_command(['cmake', - '-S', testsuitesrcdir, - '-B', testsuitebuilddir, - '-G', 'Ninja', - '-DCMAKE_BUILD_TYPE=Release', - f'-DCMAKE_C_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang', - f'-DCMAKE_CXX_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang++', - f'-DTEST_SUITE_LIT={os.path.abspath(llvmbuilddir)}/bin/llvm-lit', - f'-DTEST_SUITE_LLVM_SIZE={os.path.abspath(llvmbuilddir)}/bin/llvm-size', - "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", - "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", - '-DLLVM_LIT_ARGS=-sv{jobsarg};-o;report.json'] + with w.step("configure-testsuite", halt_on_fail=True): + jobsarg = f";-j{w.jobs}" if w.jobs else "" + w.run_command( + [ + "cmake", + f"-S{testsuitesrcdir}", + f"-B{testsuitebuilddir}", + "-GNinja", + "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_C_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang", + f"-DCMAKE_CXX_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang++", + f"-DTEST_SUITE_LIT={os.path.abspath(llvmbuilddir)}/bin/llvm-lit", + f"-DTEST_SUITE_LLVM_SIZE={os.path.abspath(llvmbuilddir)}/bin/llvm-size", + "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", + "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", + "-DLLVM_LIT_ARGS=-sv{jobsarg};-o;report.json", + ] ) - with w.step('build-testsuite', halt_on_fail=True): - w. run_ninja( builddir=testsuitebuilddir) - - with w.step('check-testsuite'): - w.run_ninja( ['check'], builddir=testsuitebuilddir) + with w.step("build-testsuite", halt_on_fail=True): + w.run_ninja(builddir=testsuitebuilddir) + with w.step("check-testsuite"): + w.run_ninja(["check"], builddir=testsuitebuilddir) From 06d82f0efc1e21fe782eea29c70411968ed32f2e Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Mon, 10 Nov 2025 00:11:51 +0100 Subject: [PATCH 10/10] Build fix --- .ci/buildbot/worker.py | 713 +++++++++++----------- polly/ci/polly-x86_64-linux-test-suite.py | 8 +- 2 files changed, 363 insertions(+), 358 deletions(-) diff --git a/.ci/buildbot/worker.py b/.ci/buildbot/worker.py index 0c1c538babc6f..81b393b0542e8 100644 --- a/.ci/buildbot/worker.py +++ b/.ci/buildbot/worker.py @@ -1,354 +1,359 @@ -import argparse -import filecmp -import os -import pathlib -import re -import shlex -import shutil -import subprocess -import sys -import traceback -from contextlib import contextmanager - -_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') -_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile("[ \t\n]") - - -def _shquote_windows(txt): - """shlex.quote for Windows cmd.exe""" - txt = txt.replace("%", "%%") - quoted = re.sub(_SHQUOTE_WINDOWS_ESCAPEDCHARS, r"\\\1", txt) - if len(quoted) == len(txt) and not _SHQUOTE_WINDOWS_QUOTEDCHARS.search(txt): - return txt - else: - return '"' + quoted + '"' - - -def shjoin(args): - """Convert a list of shell arguments to an appropriately quoted string.""" - if os.name in set(("nt", "os2", "ce")): - return " ".join(map(_shquote_windows, args)) - else: - return shlex.join(args) - - -def report(msg): - """Emit a message to the build log. Appears in red font. Lines surrounded by @@@ may be interpreted as meta-instructions.""" - print(msg, file=sys.stderr, flush=True) - - -def run_command(cmd, shell=False, **kwargs): - """Report which command is being run, then execute it using subprocess.check_call.""" - report(f"Running: {cmd if shell else shjoin(cmd)}") - sys.stderr.flush() - subprocess.check_call(cmd, shell=shell, **kwargs) - - -def _remove_readonly(func, path, _): - """Clear the readonly bit and reattempt the removal.""" - os.chmod(path, os.stat.S_IWRITE) - func(path) - - -def rmtree(path): - """ - Remove directory path and all its subdirectories. Includes a workaround for Windows where shutil.rmtree errors on read-only files. - - Taken from official Pythons docs - https://docs.python.org/3/library/shutil.html#rmtree-example - """ - shutil.rmtree(path, onexc=_remove_readonly) - - -def checkout(giturl, sourcepath): - """ - Use git to checkout the remote repository giturl at local directory sourcepath. - - If the repository already exists, clear all local changes and check out the latest main branch. - """ - if not os.path.exists(sourcepath): - run_command(["git", "clone", giturl, sourcepath]) - - # Reset repository state no matter what there was before - run_command(["git", "-C", sourcepath, "stash", "--all"]) - run_command(["git", "-C", sourcepath, "stash", "clear"]) - - # Fetch and checkout the newest - run_command(["git", "-C", sourcepath, "fetch", "origin"]) - run_command(["git", "-C", sourcepath, "checkout", "origin/main", "--detach"]) - - -@contextmanager -def step(step_name, halt_on_fail=False): - """Report a new build step being started. - - Use like this:: - with step("greet-step"): - report("Hello World!") - """ - # Barrier to separate stdio output for the the previous step - sys.stderr.flush() - sys.stdout.flush() - - report(f"@@@BUILD_STEP {step_name}@@@") - if halt_on_fail: - report("@@@HALT_ON_FAILURE@@@") - try: - yield - except Exception as e: - if isinstance(e, subprocess.CalledProcessError): - report(f"{shjoin(e.cmd)} exited with return code {e.returncode}.") - report("@@@STEP_FAILURE@@@") - else: - traceback.print_exc() - report("@@@STEP_EXCEPTION@@@") - if halt_on_fail: - # Do not continue with the next steps, but allow except/finally blocks to execute - raise e - - -class Worker: - """Helper class to keep context in a worker.run() environment""" - - def __init__(self, args, clean, clobber, workdir, jobs, cachefile, llvmsrcroot): - self.args = args - self.clean = clean - self.clobber = clobber - self.workdir = workdir - self.jobs = jobs - self.cachefile = cachefile - self.llvmsrcroot = llvmsrcroot - - def in_llvmsrc(self, path): - """Convert a path in the llvm-project source checkout to an absolute path""" - return os.path.join(self.llvmsrcroot, path) - - def in_workdir(self, path): - """Convert a path in the workdir to an absolute path""" - return os.path.join(self.workdir, path) - - def run_ninja( - self, targets: list = [], *, builddir, ccache_stats: bool = False, **kwargs - ): - """Run ninja in builddir. If self.jobs is set, automatically adds an -j option to set the number of parallel jobs. - - Parameters - ---------- - targets : list - List of build targets; build the default target 'all' if list is empty - builddir - Directory of the build.ninja file - ccache_stats : bool - If true, also emit ccache statistics when finishing the build - """ - cmd = ["ninja"] - if builddir is not None: - cmd += ["-C", builddir] - cmd += targets - if self.jobs: - cmd.append(f"-j{self.jobs}") - if ccache_stats: - run_command(["ccache", "-z"]) - try: - run_command(cmd, **kwargs) - finally: - # TODO: Pipe to stderr to separate from build log itself - run_command(["ccache", "-sv"]) - else: - run_command(cmd, **kwargs) - - @contextmanager - def step(self, step_name, halt_on_fail=False): - """Convenience wrapper for step()""" - with step(step_name, halt_on_fail=halt_on_fail) as s: - yield s - - def run_command(self, *args, **kwargs): - """Convenience wrapper for run_command()""" - return run_command(*args, **kwargs) - - def rmtree(self, *args, **kwargs): - """Convenience wrapper for rmtree()""" - return rmtree(*args, *kwargs) - - def checkout(self, giturl, sourcepath): - """Convenience wrapper for checkout()""" - return checkout(giturl, sourcepath) - - -def convert_bool(v): - """Convert input to bool type - - Use to convert the value of bool environment variables. Specifically, the buildbot master sets 'false' to build properties, which by default Python would interpret as true-ish. - """ - match v: - case None: - return False - case bool(b): - return b - case str(s): - return not s.strip().upper() in ["", "0", "N", "NO", "FALSE", "OFF"] - case _: - return bool(v) - - -def relative_if_possible(path, relative_to): - """Like os.path.relpath, but does not fail if path is not a parent of relative_to; keeps the original path in that case""" - path = os.path.normpath(path) - if not os.path.isabs(path): - # Path is already relative (assumed to relative_to) - return path - try: - result = os.path.relpath(path, start=relative_to) - return result if result else path - except ValueError: - return path - - -@contextmanager -def run( - scriptpath, - llvmsrcroot, - parser=None, - clobberpaths=[], - workerjobs=None, - always_clobber=False, -): - """Runs the boilerplate for a ScriptedBuilder buildbot. It is not necessary to use this function (one can also all run_command() etc. directly), but allows for some more flexibility and safety checks. Arguments passed to this function represent the worker configuration. - - We use the term 'clean' for resetting the worker to an empty state. This involves deleting ${prefix}/llvm.src as well as ${prefix}/build. - The term 'clobber' means deleting build artifacts, but not already downloaded git repositories. Build artifacts including build- and install-directories, but not source directories. Changes in the llvm.src directory will be reset before the next build anyway. Clobber is necessary if the build instructions change. Otherwise, we try an incremental build. We consider 'clean' to imply 'clean_obj'. - - A buildbot worker will invoke this script using this directory structure, where ${prefix} is a dedicated directory for this builder: - ${prefix}/llvm.src # Checkout location for the llvm-source - ${prefix}/build # cwd when launching the build script - - The build script is called with --workdir=. parameter, i.e. the build artifacts are written into ${prefix}/build. When cleaning, the worker (NOT the build script) will delete ${prefix}/llvm.src; Deleting any contents of ${prefix}/build is to be done by the builder script, e.g. by this function. The builder script can choose to not delete the complete workdir, e.g. additional source checkouts such as the llvm-test-suite. - - The buildbot master will set the 'clean' build property and the environment variable BUILDBOT_CLEAN when in the GUI the option "Clean source code and build directory" is checked by the user. The 'clean_obj' build property and the BUILDBOT_CLEAN_OBJ environment variable will be set when either the "Clean build directory" GUI option is set, or the master detects a change to a CMakeLists.txt or *.cmake file. - - Parameters - ---------- - scriptpath - Pass __file__ from the main builder script. - llvmsrcroot - Absolute path to the llvm-project source checkout. Since the builder script is supposed to be a part of llvm-project itself, the builder script can compute it from __file__. - parser - Use this argparse.ArgumentParser instead of creating a new one. Allows adding additional command line switched in addition to the pre-defined ones. Build script are encouraged to apply the pre-defined switches. - clobberpaths - Directories relative to workdir that need to be deleted if the build configuration changes (due to changes of CMakeLists.txt or changes of configuration parameters). Typically, only source checkouts are not deleted. - workerjobs - Default number of build and test jobs; If set, expected to be the number of jobs of the actual buildbot worker that executes this script. Can be overridden using the --jobs parameter so in case someone needs to reproduce this build, they can adjust the number of jobs for the reproducer platform. Alternatively, the worker can set the BUILDBOT_JOBS environment variable or keep ninja/llvm-lit defaults. - always_clobber - Always clobber the build artifacts, i.e. disable incremental builds. - """ - - assert os.path.isabs(scriptpath) - assert os.path.isabs(llvmsrcroot) - - jobs_default = None - if jobs_env := os.environ.get("BUILDBOT_JOBS"): - jobs_default = int(jobs_env) - if not jobs_default: - jobs_default = workerjobs - if not jobs_default: - jobs_default = None - - stem = pathlib.Path(scriptpath).stem - workdir_default = f"{stem}.workdir" - - parser = parser or argparse.ArgumentParser( - allow_abbrev=True, - description=f"When executed without arguments, builds the worker's LLVM build configuration in {os.path.abspath(workdir_default)}. Some build configuration parameters can be altered using the following switches:", - ) - parser.add_argument( - "--workdir", - default=workdir_default, - help="Use this dir as workdir to write the build artifact into. --workdir=. uses the current directory.\nWarning: This directory might be deleted", - ) - parser.add_argument( - "--cachefile", - default=relative_if_possible( - pathlib.Path(scriptpath).with_suffix(".cmake"), llvmsrcroot - ), - help="File containing the initial values for the CMakeCache.txt for the llvm build.", - ) - parser.add_argument( - "--clean", - type=bool, - default=convert_bool(os.environ.get("BUILDBOT_CLEAN")), - help="Delete the entire workdir before starting the build, including source directories", - ) - parser.add_argument( - "--clobber", - type=bool, - default=always_clobber - or convert_bool(os.environ.get("BUILDBOT_CLOBBER")) - or convert_bool(os.environ.get("BUILDBOT_CLEAN_OBJ")), - help="Delete build artifacts before starting the build", - ) - parser.add_argument( - "--jobs", "-j", default=jobs_default, help="Number of build- and test-jobs" - ) - args = parser.parse_args() - - workdir = args.workdir - clean = args.clean - clobber = args.clobber - cachefile = os.path.join(llvmsrcroot, args.cachefile) - oldcwd = os.getcwd() - - prevcachepath = os.path.join(workdir, "prevcache.cmake") - if cachefile and os.path.exists(prevcachepath): - # Force clobber if cache file has changed; a new cachefile does not override entries already present in CMakeCache.txt - if not filecmp.cmp( - os.path.join(llvmsrcroot, args.cachefile), prevcachepath, shallow=False - ): - clobber = True - - w = Worker( - args, - clean=clean, - clobber=clobber, - workdir=workdir, - jobs=args.jobs, - cachefile=cachefile, - llvmsrcroot=llvmsrcroot, - ) - - # Safety check - parentdir = os.path.dirname(scriptpath) - while True: - if os.path.samefile(parentdir, workdir): - raise Exception( - f"Cannot use {workdir} as workdir; a '--clean' build would rmtree the llvm-project source as well" - ) - newparentdir = os.path.dirname(parentdir) - if newparentdir == parentdir: - break - parentdir = newparentdir - - if clean: - # Ensure that the cwd is not the directory we are going to delete. This would not work under Windows. We will chdir to workdir later anyway. - os.chdir("/") - - with w.step(f"clean"): - rmtree(workdir) - elif clobber: - with w.step(f"clobber"): - for d in clobberpaths: - rmtree(os.path.join(workdir, d)) - os.path.unlink(prevcachepath) - - os.makedirs(workdir, exist_ok=True) - os.chdir(workdir) - - # Remember used cachefile in case it changes - if cachefile: - shutil.copy( - os.path.join(oldcwd, llvmsrcroot, args.cachefile), - os.path.join(oldcwd, prevcachepath), - ) - - os.environ["NINJA_STATUS"] = "[%p/%es :: %u->%r->%f (of %t)] " - yield w +import argparse +import filecmp +import os +import pathlib +import re +import shlex +import shutil +import subprocess +import sys +import traceback +from contextlib import contextmanager + +_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') +_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile("[ \t\n]") + + +def _shquote_windows(txt): + """shlex.quote for Windows cmd.exe""" + txt = txt.replace("%", "%%") + quoted = re.sub(_SHQUOTE_WINDOWS_ESCAPEDCHARS, r"\\\1", txt) + if len(quoted) == len(txt) and not _SHQUOTE_WINDOWS_QUOTEDCHARS.search(txt): + return txt + else: + return '"' + quoted + '"' + + +def shjoin(args): + """Convert a list of shell arguments to an appropriately quoted string.""" + if os.name in set(("nt", "os2", "ce")): + return " ".join(map(_shquote_windows, args)) + else: + return shlex.join(args) + + +def report(msg): + """Emit a message to the build log. Appears in red font. Lines surrounded by @@@ may be interpreted as meta-instructions.""" + print(msg, file=sys.stderr, flush=True) + + +def run_command(cmd, shell=False, **kwargs): + """Report which command is being run, then execute it using subprocess.check_call.""" + report(f"Running: {cmd if shell else shjoin(cmd)}") + sys.stderr.flush() + subprocess.check_call(cmd, shell=shell, **kwargs) + + +def _remove_readonly(func, path, _): + """Clear the readonly bit and reattempt the removal.""" + os.chmod(path, os.stat.S_IWRITE) + func(path) + + +def rmtree(path): + """ + Remove directory path and all its subdirectories. Includes a workaround for Windows where shutil.rmtree errors on read-only files. + + Taken from official Pythons docs + https://docs.python.org/3/library/shutil.html#rmtree-example + """ + shutil.rmtree(path, onexc=_remove_readonly) + + +def checkout(giturl, sourcepath): + """ + Use git to checkout the remote repository giturl at local directory sourcepath. + + If the repository already exists, clear all local changes and check out the latest main branch. + """ + if not os.path.exists(sourcepath): + run_command(["git", "clone", giturl, sourcepath]) + + # Reset repository state no matter what there was before + run_command(["git", "-C", sourcepath, "stash", "--all"]) + run_command(["git", "-C", sourcepath, "stash", "clear"]) + + # Fetch and checkout the newest + run_command(["git", "-C", sourcepath, "fetch", "origin"]) + run_command(["git", "-C", sourcepath, "checkout", "origin/main", "--detach"]) + + +@contextmanager +def step(step_name, halt_on_fail=False): + """Report a new build step being started. + + Use like this:: + with step("greet-step"): + report("Hello World!") + """ + # Barrier to separate stdio output for the the previous step + sys.stderr.flush() + sys.stdout.flush() + + report(f"@@@BUILD_STEP {step_name}@@@") + if halt_on_fail: + report("@@@HALT_ON_FAILURE@@@") + try: + yield + except Exception as e: + if isinstance(e, subprocess.CalledProcessError): + report(f"{shjoin(e.cmd)} exited with return code {e.returncode}.") + report("@@@STEP_FAILURE@@@") + else: + traceback.print_exc() + report("@@@STEP_EXCEPTION@@@") + if halt_on_fail: + # Do not continue with the next steps, but allow except/finally blocks to execute + raise e + + +class Worker: + """Helper class to keep context in a worker.run() environment""" + + def __init__(self, args, clean, clobber, workdir, jobs, cachefile, llvmsrcroot): + self.args = args + self.clean = clean + self.clobber = clobber + self.workdir = workdir + self.jobs = jobs + self.cachefile = cachefile + self.llvmsrcroot = llvmsrcroot + + def in_llvmsrc(self, path): + """Convert a path in the llvm-project source checkout to an absolute path""" + return os.path.join(self.llvmsrcroot, path) + + def in_workdir(self, path): + """Convert a path in the workdir to an absolute path""" + return os.path.join(self.workdir, path) + + def run_ninja( + self, targets: list = [], *, builddir, ccache_stats: bool = False, **kwargs + ): + """Run ninja in builddir. If self.jobs is set, automatically adds an -j option to set the number of parallel jobs. + + Parameters + ---------- + targets : list + List of build targets; build the default target 'all' if list is empty + builddir + Directory of the build.ninja file + ccache_stats : bool + If true, also emit ccache statistics when finishing the build + """ + cmd = ["ninja"] + if builddir is not None: + cmd += ["-C", builddir] + cmd += targets + if self.jobs: + cmd.append(f"-j{self.jobs}") + if ccache_stats: + run_command(["ccache", "-z"]) + try: + run_command(cmd, **kwargs) + finally: + # TODO: Pipe to stderr to separate from build log itself + run_command(["ccache", "-sv"]) + else: + run_command(cmd, **kwargs) + + @contextmanager + def step(self, step_name, halt_on_fail=False): + """Convenience wrapper for step()""" + with step(step_name, halt_on_fail=halt_on_fail) as s: + yield s + + def run_command(self, *args, **kwargs): + """Convenience wrapper for run_command()""" + return run_command(*args, **kwargs) + + def rmtree(self, *args, **kwargs): + """Convenience wrapper for rmtree()""" + return rmtree(*args, *kwargs) + + def checkout(self, giturl, sourcepath): + """Convenience wrapper for checkout()""" + return checkout(giturl, sourcepath) + + +def convert_bool(v): + """Convert input to bool type + + Use to convert the value of bool environment variables. Specifically, the buildbot master sets 'false' to build properties, which by default Python would interpret as true-ish. + """ + match v: + case None: + return False + case bool(b): + return b + case str(s): + return not s.strip().upper() in ["", "0", "N", "NO", "FALSE", "OFF"] + case _: + return bool(v) + + +def relative_if_possible(path, relative_to): + """Like os.path.relpath, but does not fail if path is not a parent of relative_to; keeps the original path in that case""" + path = os.path.normpath(path) + if not os.path.isabs(path): + # Path is already relative (assumed to relative_to) + return path + try: + result = os.path.relpath(path, start=relative_to) + return result if result else path + except ValueError: + return path + + +@contextmanager +def run( + scriptpath, + llvmsrcroot, + parser=None, + clobberpaths=[], + workerjobs=None, + always_clobber=False, +): + """Runs the boilerplate for a ScriptedBuilder buildbot. It is not necessary to use this function (one can also all run_command() etc. directly), but allows for some more flexibility and safety checks. Arguments passed to this function represent the worker configuration. + + We use the term 'clean' for resetting the worker to an empty state. This involves deleting ${prefix}/llvm.src as well as ${prefix}/build. + The term 'clobber' means deleting build artifacts, but not already downloaded git repositories. Build artifacts including build- and install-directories, but not source directories. Changes in the llvm.src directory will be reset before the next build anyway. Clobber is necessary if the build instructions change. Otherwise, we try an incremental build. We consider 'clean' to imply 'clean_obj'. + + A buildbot worker will invoke this script using this directory structure, where ${prefix} is a dedicated directory for this builder: + ${prefix}/llvm.src # Checkout location for the llvm-source + ${prefix}/build # cwd when launching the build script + + The build script is called with --workdir=. parameter, i.e. the build artifacts are written into ${prefix}/build. When cleaning, the worker (NOT the build script) will delete ${prefix}/llvm.src; Deleting any contents of ${prefix}/build is to be done by the builder script, e.g. by this function. The builder script can choose to not delete the complete workdir, e.g. additional source checkouts such as the llvm-test-suite. + + The buildbot master will set the 'clean' build property and the environment variable BUILDBOT_CLEAN when in the GUI the option "Clean source code and build directory" is checked by the user. The 'clean_obj' build property and the BUILDBOT_CLEAN_OBJ environment variable will be set when either the "Clean build directory" GUI option is set, or the master detects a change to a CMakeLists.txt or *.cmake file. + + Parameters + ---------- + scriptpath + Pass __file__ from the main builder script. + llvmsrcroot + Absolute path to the llvm-project source checkout. Since the builder script is supposed to be a part of llvm-project itself, the builder script can compute it from __file__. + parser + Use this argparse.ArgumentParser instead of creating a new one. Allows adding additional command line switched in addition to the pre-defined ones. Build script are encouraged to apply the pre-defined switches. + clobberpaths + Directories relative to workdir that need to be deleted if the build configuration changes (due to changes of CMakeLists.txt or changes of configuration parameters). Typically, only source checkouts are not deleted. + workerjobs + Default number of build and test jobs; If set, expected to be the number of jobs of the actual buildbot worker that executes this script. Can be overridden using the --jobs parameter so in case someone needs to reproduce this build, they can adjust the number of jobs for the reproducer platform. Alternatively, the worker can set the BUILDBOT_JOBS environment variable or keep ninja/llvm-lit defaults. + always_clobber + Always clobber the build artifacts, i.e. disable incremental builds. + """ + + + scriptpath = os.path.abspath(scriptpath) + llvmsrcroot = os.path.abspath(llvmsrcroot) + stem = pathlib.Path(scriptpath).stem + workdir_default = f"{stem}.workdir" + + + jobs_default = None + if jobs_env := os.environ.get("BUILDBOT_JOBS"): + jobs_default = int(jobs_env) + if not jobs_default: + jobs_default = workerjobs + if not jobs_default: + jobs_default = None + + parser = parser or argparse.ArgumentParser( + allow_abbrev=True, + description=f"When executed without arguments, builds the worker's LLVM build configuration in {os.path.abspath(workdir_default)}. Some build configuration parameters can be altered using the following switches:", + ) + parser.add_argument( + "--workdir", + default=workdir_default, + help="Use this dir as workdir to write the build artifact into. --workdir=. uses the current directory.\nWarning: This directory might be deleted", + ) + parser.add_argument( + "--cachefile", + default=relative_if_possible( + pathlib.Path(scriptpath).with_suffix(".cmake"), llvmsrcroot + ), + help="File containing the initial values for the CMakeCache.txt for the llvm build.", + ) + parser.add_argument( + "--clean", + type=bool, + default=convert_bool(os.environ.get("BUILDBOT_CLEAN")), + help="Delete the entire workdir before starting the build, including source directories", + ) + parser.add_argument( + "--clobber", + type=bool, + default=always_clobber + or convert_bool(os.environ.get("BUILDBOT_CLOBBER")) + or convert_bool(os.environ.get("BUILDBOT_CLEAN_OBJ")), + help="Delete build artifacts before starting the build", + ) + parser.add_argument( + "--jobs", "-j", default=jobs_default, help="Number of build- and test-jobs" + ) + args = parser.parse_args() + + workdir = os.path.abspath( args.workdir) + clean = args.clean + clobber = args.clobber + cachefile = os.path.join(llvmsrcroot, args.cachefile) + oldcwd = os.getcwd() + + prevcachepath = os.path.join(workdir, "prevcache.cmake") + if cachefile and os.path.exists(prevcachepath): + # Force clobber if cache file has changed; a new cachefile does not override entries already present in CMakeCache.txt + if not filecmp.cmp( + os.path.join(llvmsrcroot, args.cachefile), prevcachepath, shallow=False + ): + clobber = True + + + # Safety check + parentdir = os.path.dirname(scriptpath) + while True: + if os.path.samefile(parentdir, workdir): + raise Exception( + f"Cannot use {args.workdir} as workdir; a '--clean' build would rmtree the llvm-project source in {parentdir} as well" + ) + newparentdir = os.path.dirname(parentdir) + if newparentdir == parentdir: + break + parentdir = newparentdir + + + w = Worker( + args, + clean=clean, + clobber=clobber, + workdir=workdir, + jobs=args.jobs, + cachefile=cachefile, + llvmsrcroot=llvmsrcroot, + ) + + + + if clean: + # Ensure that the cwd is not the directory we are going to delete. This would not work under Windows. We will chdir to workdir later anyway. + os.chdir("/") + + with w.step(f"clean"): + rmtree(workdir) + elif clobber: + with w.step(f"clobber"): + for d in clobberpaths: + rmtree(os.path.join(workdir, d)) + os.path.unlink(prevcachepath) + + os.makedirs(workdir, exist_ok=True) + os.chdir(workdir) + + # Remember used cachefile in case it changes + if cachefile: + shutil.copy( + os.path.join(oldcwd, llvmsrcroot, args.cachefile), + os.path.join(oldcwd, prevcachepath), + ) + + os.environ["NINJA_STATUS"] = "[%p/%es :: %u->%r->%f (of %t)] " + yield w diff --git a/polly/ci/polly-x86_64-linux-test-suite.py b/polly/ci/polly-x86_64-linux-test-suite.py index e50c39e9757b0..be60798847777 100644 --- a/polly/ci/polly-x86_64-linux-test-suite.py +++ b/polly/ci/polly-x86_64-linux-test-suite.py @@ -22,8 +22,8 @@ with w.step("configure-llvm", halt_on_fail=True): cmakecmd = [ "cmake", - f'-S{w.in_llvmsrc("llvm")}', - "-B{llvmbuilddir}", + f"-S{w.in_llvmsrc('llvm')}", + f"-B{llvmbuilddir}", "-GNinja", f"-C{w.in_llvmsrc(w.cachefile)}", f"-DCMAKE_INSTALL_PREFIX={llvminstalldir}", @@ -36,7 +36,7 @@ w.run_ninja(builddir=llvmbuilddir, ccache_stats=True) with w.step("check-polly"): - w.run_ninja(["check-polly"], builddir=llvmbuilddir, targets=[]) + w.run_ninja(["check-polly"], builddir=llvmbuilddir) with w.step("install-llvm", halt_on_fail=True): w.run_ninja(["install"], builddir=llvmbuilddir) @@ -59,7 +59,7 @@ f"-DTEST_SUITE_LLVM_SIZE={os.path.abspath(llvmbuilddir)}/bin/llvm-size", "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", - "-DLLVM_LIT_ARGS=-sv{jobsarg};-o;report.json", + f"-DLLVM_LIT_ARGS=-sv{jobsarg};-o;report.json", ] )