diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py index 23fc7585d..24423665c 100644 --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -3,6 +3,7 @@ from buildbot.plugins import util, steps from zorg.buildbot.builders import ClangBuilder +from zorg.buildbot.builders import ScriptedBuilder from zorg.buildbot.builders import FlangBuilder from zorg.buildbot.builders import PollyBuilder from zorg.buildbot.builders import LLDBBuilder @@ -1310,28 +1311,9 @@ def collapseRequestsDoxygen(master, builder, req1, req2): 'tags' : ["polly"], 'workernames' : ["polly-x86_64-fdcserver", "minipc-1050ti-linux"], 'builddir': "polly-x86_64-linux-test-suite", - 'factory' : PollyBuilder.getPollyBuildFactory( - clean=False, - install=False, - make='ninja', - extraCmakeArgs=[ - "-G", "Ninja", - "-DCMAKE_C_COMPILER_LAUNCHER=ccache", - "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", - "-DLLVM_ENABLE_ASSERTIONS=True", - "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'", - "-DCLANG_ENABLE_ARCMT=OFF", - "-DCLANG_ENABLE_STATIC_ANALYZER=OFF", - "-DCLANG_ENABLE_OBJC_REWRITER=OFF" - ], - testsuite=True, - extraTestsuiteCmakeArgs=[ - "-G", "Ninja", - "-DTEST_SUITE_COLLECT_COMPILE_TIME=OFF", - "-DTEST_SUITE_COLLECT_STATS=OFF", - "-DTEST_SUITE_COLLECT_CODE_SIZE=OFF", - util.Interpolate("-DTEST_SUITE_EXTERNALS_DIR=%(prop:builddir)s/../../test-suite-externals"), - ] + 'factory' : ScriptedBuilder.getScriptedBuildFactory( + "polly/ci/polly-x86_64-linux-test-suite.py", + depends_on_projects=["llvm", "clang", "polly"], )}, # AOSP builders. @@ -1462,11 +1444,11 @@ def collapseRequestsDoxygen(master, builder, req1, req2): "CMAKE_CXX_FLAGS" : "-D__OPTIMIZE__", "CMAKE_MSVC_RUNTIME_LIBRARY" : "MultiThreadedDLL", - "LLVM_ENABLE_ASSERTIONS" : "ON", + "LLVM_ENABLE_ASSERTIONS" : "ON", "LLVM_INCLUDE_BENCHMARKS" : "OFF", "LLVM_PARALLEL_LINK_JOBS" : 8, "LLVM_LIT_ARGS" : "-v -vv --threads=32 --time-tests", - + "LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS" : "ON", "LLDB_ENABLE_SWIG" : "ON ", "LLDB_ENABLE_LIBEDIT" : "OFF", @@ -3565,7 +3547,7 @@ def collapseRequestsDoxygen(master, builder, req1, req2): checkout_llvm_sources=False, script_interpreter=None, clean=True)}, - + # Builders that test the premerge configuration # These builders are specifically for running the premerge configuration # postcommit (after changes have landed in main). The configuration for @@ -3939,7 +3921,7 @@ def collapseRequestsDoxygen(master, builder, req1, req2): env = lldb_remote_linux_env.copy(), ) }, - + # PtrAuth (PAuth) builders. {'name' : "llvm-clang-ubuntu-x-aarch64-pauth", 'tags' : ["clang", "llvm", "lld", "clang-tools-extra", "compiler-rt", "libc++", "libc++abi", "libunwind", "cross", "aarch64", "pauth", "ptrauth"], diff --git a/zorg/buildbot/builders/ScriptedBuilder.py b/zorg/buildbot/builders/ScriptedBuilder.py new file mode 100644 index 000000000..e1f590889 --- /dev/null +++ b/zorg/buildbot/builders/ScriptedBuilder.py @@ -0,0 +1,112 @@ +from buildbot.plugins import util +from buildbot.steps.shell import SetProperty +from zorg.buildbot.commands.AnnotatedCommand import AnnotatedCommand +from zorg.buildbot.process.factory import LLVMBuildFactory +from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand +from buildbot.plugins import steps, util +from zorg.buildbot.commands.LitTestCommand import LitTestCommand +from zorg.buildbot.process.factory import LLVMBuildFactory +import os + + +def getScriptedBuildFactory(scriptpath, *scriptargs, depends_on_projects, always_clobber=False, + env=None, + extra_args=None, + timeout=1200, + script_interpreter="python", + warnOnWarnings=False, **kwargs): + + assert scriptpath, "Must specify a script the worker is going to execute" + assert depends_on_projects, "Must specify a set of projects; any change one of those projects will trigger a worker run" + + llvm_srcdir = "llvm.src" + #venvpath = "bbenv" + workdir = "workdir" + + + # If true, clean everything, including source dirs + def cleanBuildRequested(step): + return step.build.getProperty("clean") + + # If true, clean build products; implied if cleanBuildRequested is true + def clobberRequested(step): + return cleanBuildRequested(step) or always_clobber or step.build.getProperty("clean_obj") + + f = LLVMBuildFactory( + depends_on_projects=depends_on_projects, + llvm_srcdir=llvm_srcdir) + + + + f.addStep(steps.RemoveDirectory(name='clean-srcdir', + dir=f.monorepo_dir, + warnOnFailure=True, + doStepIf=cleanBuildRequested)) + + + f.addStep(steps.RemoveDirectory(name='clobber-workdir', + dir=f.monorepo_dir, + warnOnFailure=True, + doStepIf=clobberRequested)) + + #f.addStep(steps.RemoveDirectory(name='clean-venv-dir', + # dir=venvpath, + # warnOnFailure=True, + # doStepIf=cleanBuildRequested)) + + + + + + # Get the source code. + f.addGetSourcecodeSteps(**kwargs) + + + #if always_clobber: + # f.addStep(SetProperty(property='clean', command='echo 1', doStepIf=cleanBuildRequested)) + + + # We normally use the clean property to indicate that we want a + # clean build, but AnnotatedCommand uses the clobber property + # instead. Therefore, set clobber if clean is set to a truthy + # value. This will cause AnnotatedCommand to set + # BUILDBOT_CLOBBER=1 in the environment, which is how we + # communicate to the script that we need a clean build. + #f.addStep(SetProperty( + # property='clobber', + # command='echo 1', + # doStepIf=clobberRequested)) + + + merged_env = { + 'TERM': 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set + # anything. + merged_env.update(env) + + + + #f.addStep(ShellCommand( + # # name="create-venv", + # command=[script_interpreter, "-m", "venv", venvpath], # " --upgrade", "--upgrade-deps", + # # description="Create a venv" + #)) + #venv_interpreter = os.path.join(venvpath, 'bin', 'python') + + + command = [script_interpreter, os.path.join('..', f.monorepo_dir, scriptpath), f'--workdir=.'] # relative to build path + if always_clobber: + command += ['--clobber'] + command += [util.Interpolate(arg) for arg in scriptargs] + + f.addStep(AnnotatedCommand(name="annotate", + description="Run build script", + timeout=timeout, + haltOnFailure=True, + warnOnWarnings=warnOnWarnings, + command=command, + env=merged_env)) + + return f diff --git a/zorg/buildbot/commands/AnnotatedCommand.py b/zorg/buildbot/commands/AnnotatedCommand.py index d21f16746..5d91c2fd6 100644 --- a/zorg/buildbot/commands/AnnotatedCommand.py +++ b/zorg/buildbot/commands/AnnotatedCommand.py @@ -324,6 +324,8 @@ def __init__(self, **kwargs): 'BUILDBOT_BRANCH': util.Interpolate('%(prop:branch:-None)s'), 'BUILDBOT_BUILDERNAME': util.Interpolate('%(prop:buildername:-None)s'), 'BUILDBOT_BUILDNUMBER': util.Interpolate('%(prop:buildnumber:-None)s'), + 'BUILDBOT_CLEAN': util.Interpolate('%(prop:clean:-)s'), + 'BUILDBOT_CLEAN_OBJ': util.Interpolate('%(prop:clean_obj:-)s'), 'BUILDBOT_CLOBBER': util.Interpolate('%(prop:clobber:+1)s'), 'BUILDBOT_GOT_REVISION': util.Interpolate('%(prop:got_revision:-None)s'), 'BUILDBOT_REVISION': util.Interpolate('%(prop:revision:-None)s'),