Introduce ScriptedBuilder

buildbot/osuosl/master/config/builders.py

@@ -3,6 +3,7 @@
 from buildbot.plugins import util, steps
 
 from zorg.buildbot.builders import ClangBuilder
+from zorg.buildbot.builders import ScriptedBuilder
 from zorg.buildbot.builders import FlangBuilder
 from zorg.buildbot.builders import PollyBuilder
 from zorg.buildbot.builders import LLDBBuilder
@@ -1310,28 +1311,9 @@ def collapseRequestsDoxygen(master, builder, req1, req2):
     'tags'  : ["polly"],
     'workernames' : ["polly-x86_64-fdcserver", "minipc-1050ti-linux"],
     'builddir': "polly-x86_64-linux-test-suite",
-    'factory' : PollyBuilder.getPollyBuildFactory(
-                    clean=False,
-                    install=False,
-                    make='ninja',
-                    extraCmakeArgs=[
-                        "-G", "Ninja",
-                        "-DCMAKE_C_COMPILER_LAUNCHER=ccache",
-                        "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
-                        "-DLLVM_ENABLE_ASSERTIONS=True",
-                        "-DLLVM_TARGETS_TO_BUILD='X86;NVPTX'",
-                        "-DCLANG_ENABLE_ARCMT=OFF",
-                        "-DCLANG_ENABLE_STATIC_ANALYZER=OFF",
-                        "-DCLANG_ENABLE_OBJC_REWRITER=OFF"
-                        ],
-                    testsuite=True,
-                    extraTestsuiteCmakeArgs=[
-                        "-G", "Ninja",
-                        "-DTEST_SUITE_COLLECT_COMPILE_TIME=OFF",
-                        "-DTEST_SUITE_COLLECT_STATS=OFF",
-                        "-DTEST_SUITE_COLLECT_CODE_SIZE=OFF",
-                        util.Interpolate("-DTEST_SUITE_EXTERNALS_DIR=%(prop:builddir)s/../../test-suite-externals"),
-                      ]
+    'factory' : ScriptedBuilder.getScriptedBuildFactory(
+                      "polly/ci/polly-x86_64-linux-test-suite.py",
+                      depends_on_projects=["llvm", "clang", "polly"],
                     )},
 
 # AOSP builders.
@@ -1462,11 +1444,11 @@ def collapseRequestsDoxygen(master, builder, req1, req2):
                         "CMAKE_CXX_FLAGS"               : "-D__OPTIMIZE__",
                         "CMAKE_MSVC_RUNTIME_LIBRARY"    : "MultiThreadedDLL",
 
-                        "LLVM_ENABLE_ASSERTIONS"        : "ON",    
+                        "LLVM_ENABLE_ASSERTIONS"        : "ON",
                         "LLVM_INCLUDE_BENCHMARKS"       : "OFF",
                         "LLVM_PARALLEL_LINK_JOBS"       : 8,
                         "LLVM_LIT_ARGS"                 : "-v -vv --threads=32 --time-tests",
-                        
+
                         "LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS" : "ON",
                         "LLDB_ENABLE_SWIG"              : "ON ",
                         "LLDB_ENABLE_LIBEDIT"           : "OFF",
@@ -3565,7 +3547,7 @@ def collapseRequestsDoxygen(master, builder, req1, req2):
                     checkout_llvm_sources=False,
                     script_interpreter=None,
                     clean=True)},
-    
+
     # Builders that test the premerge configuration
     # These builders are specifically for running the premerge configuration
     # postcommit (after changes have landed in main). The configuration for
@@ -3939,7 +3921,7 @@ def collapseRequestsDoxygen(master, builder, req1, req2):
                     env = lldb_remote_linux_env.copy(),
                 )
         },
-        
+
     # PtrAuth (PAuth) builders.
     {'name' : "llvm-clang-ubuntu-x-aarch64-pauth",
     'tags'  : ["clang", "llvm", "lld", "clang-tools-extra", "compiler-rt", "libc++", "libc++abi", "libunwind", "cross", "aarch64", "pauth", "ptrauth"],

zorg/buildbot/builders/ScriptedBuilder.py

@@ -0,0 +1,112 @@
+from buildbot.plugins import util
+from buildbot.steps.shell import SetProperty
+from zorg.buildbot.commands.AnnotatedCommand import AnnotatedCommand
+from zorg.buildbot.process.factory import LLVMBuildFactory
+from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand
+from buildbot.plugins import steps, util
+from zorg.buildbot.commands.LitTestCommand import LitTestCommand
+from zorg.buildbot.process.factory import LLVMBuildFactory
+import os
+
+
+def getScriptedBuildFactory(scriptpath, *scriptargs, depends_on_projects, always_clobber=False,
+	    env=None,
+	    extra_args=None,
+	    timeout=1200,
+	    script_interpreter="python",
+	    warnOnWarnings=False, **kwargs):
+
+    assert scriptpath, "Must specify a script the worker is going to execute"
+    assert depends_on_projects, "Must specify a set of projects; any change one of those projects will trigger a worker run"
+
+    llvm_srcdir = "llvm.src"
+    #venvpath = "bbenv"
+    workdir = "workdir"
+
+
+    # If true, clean everything, including source dirs
+    def cleanBuildRequested(step):
+        return step.build.getProperty("clean")
+
+    # If true, clean build products; implied if cleanBuildRequested is true
+    def clobberRequested(step):
+        return cleanBuildRequested(step) or always_clobber or step.build.getProperty("clean_obj")
+
+    f = LLVMBuildFactory(
+        depends_on_projects=depends_on_projects,
+        llvm_srcdir=llvm_srcdir)
+
+
+
+    f.addStep(steps.RemoveDirectory(name='clean-srcdir',
+                           dir=f.monorepo_dir,
+                           warnOnFailure=True,
+                           doStepIf=cleanBuildRequested))
+
+
+    f.addStep(steps.RemoveDirectory(name='clobber-workdir',
+                           dir=f.monorepo_dir,
+                           warnOnFailure=True,
+                           doStepIf=clobberRequested))
+
+    #f.addStep(steps.RemoveDirectory(name='clean-venv-dir',
+    #                       dir=venvpath,
+    #                       warnOnFailure=True,
+    #                       doStepIf=cleanBuildRequested))
+
+
+
+
+
+    # Get the source code.
+    f.addGetSourcecodeSteps(**kwargs)
+
+
+    #if always_clobber:
+    #    f.addStep(SetProperty(property='clean', command='echo 1', doStepIf=cleanBuildRequested))
+
+
+    # We normally use the clean property to indicate that we want a
+    # clean build, but AnnotatedCommand uses the clobber property
+    # instead. Therefore, set clobber if clean is set to a truthy
+    # value.  This will cause AnnotatedCommand to set
+    # BUILDBOT_CLOBBER=1 in the environment, which is how we
+    # communicate to the script that we need a clean build.
+    #f.addStep(SetProperty(
+    #    property='clobber',
+    #    command='echo 1',
+    #    doStepIf=clobberRequested))
+
+
+    merged_env = {
+        'TERM': 'dumb'  # Be cautious and disable color output from all tools.
+    }
+    if env is not None:
+        # Overwrite pre-set items with the given ones, so user can set
+        # anything.
+        merged_env.update(env)
+
+
+
+    #f.addStep(ShellCommand(
+    #      #  name="create-venv",
+    #        command=[script_interpreter, "-m", "venv", venvpath], # " --upgrade", "--upgrade-deps",
+    #      #  description="Create a venv"
+    #))
+    #venv_interpreter = os.path.join(venvpath, 'bin', 'python')
+
+
+    command = [script_interpreter, os.path.join('..',  f.monorepo_dir, scriptpath), f'--workdir=.'] # relative to build path
+    if always_clobber:
+        command += ['--clobber']
+    command += [util.Interpolate(arg) for arg in scriptargs]
+
+    f.addStep(AnnotatedCommand(name="annotate",
+                               description="Run build script",
+                               timeout=timeout,
+                               haltOnFailure=True,
+                               warnOnWarnings=warnOnWarnings,
+                               command=command,
+                               env=merged_env))
+
+    return f

zorg/buildbot/commands/AnnotatedCommand.py

@@ -324,6 +324,8 @@ def __init__(self, **kwargs):
             'BUILDBOT_BRANCH':          util.Interpolate('%(prop:branch:-None)s'),
             'BUILDBOT_BUILDERNAME':     util.Interpolate('%(prop:buildername:-None)s'),
             'BUILDBOT_BUILDNUMBER':     util.Interpolate('%(prop:buildnumber:-None)s'),
+            'BUILDBOT_CLEAN':           util.Interpolate('%(prop:clean:-)s'),
+            'BUILDBOT_CLEAN_OBJ':       util.Interpolate('%(prop:clean_obj:-)s'),
             'BUILDBOT_CLOBBER':         util.Interpolate('%(prop:clobber:+1)s'),
             'BUILDBOT_GOT_REVISION':    util.Interpolate('%(prop:got_revision:-None)s'),
             'BUILDBOT_REVISION':        util.Interpolate('%(prop:revision:-None)s'),