Invoke build tool via cmake --build in test-suite (#21)

Instead of directly invoking make, this patch teaches test-suite to call `cmake --build ...` instead, which abstracts away the underlying build tool.

This fixes building when the CMake generator is overriden, e.g. with `CMAKE_GENERATOR=Ninja lnt runtest test-suite ...`

The existing --use-make argument is kept and passed to CMAKE_MAKE_PROGRAM if set for compatibility.

Author: lukel97

lnt/tests/test_suite.py

@@ -225,9 +225,6 @@ def run_test(self, opts):
         opts.cmake = resolve_command_path(opts.cmake)
         if not isexecfile(opts.cmake):
             self._fatal("CMake tool not found (looked for %s)" % opts.cmake)
-        opts.make = resolve_command_path(opts.make)
-        if not isexecfile(opts.make):
-            self._fatal("Make tool not found (looked for %s)" % opts.make)
         opts.lit = resolve_command_path(opts.lit)
         if not isexecfile(opts.lit):
             self._fatal("LIT tool not found (looked for %s)" % opts.lit)
@@ -387,7 +384,7 @@ def run(self, cmake_vars, compile=True, test=True, profile=False):
         if self.compiled and compile:
             self._clean(self._base_path)
         if not self.compiled or compile or self.opts.pgo:
-            self._make(self._base_path)
+            self._build(self._base_path)
             self._install_benchmark(self._base_path)
             self.compiled = True
 
@@ -431,14 +428,14 @@ def _check_output(self, *args, **kwargs):
         return output
 
     def _clean(self, path):
-        make_cmd = self.opts.make
+        cmake_cmd = self.opts.cmake
 
         subdir = path
         if self.opts.only_test:
             components = [path] + [self.opts.only_test[0]]
             subdir = os.path.join(*components)
 
-        self._check_call([make_cmd, 'clean'],
+        self._check_call([cmake_cmd, '--build', '.', '-t', 'clean'],
                          cwd=subdir)
 
     def _configure(self, path, extra_cmake_defs=[], execute=True):
@@ -449,6 +446,8 @@ def _configure(self, path, extra_cmake_defs=[], execute=True):
             defs['CMAKE_C_COMPILER'] = self.opts.cc
         if self.opts.cxx:
             defs['CMAKE_CXX_COMPILER'] = self.opts.cxx
+        if self.opts.make:
+            defs['CMAKE_MAKE_PROGRAM'] = self.opts.make
 
         cmake_build_types = ('DEBUG', 'MINSIZEREL', 'RELEASE',
                              'RELWITHDEBINFO')
@@ -560,12 +559,12 @@ def _collect_pgo(self, path):
                       "TEST_SUITE_RUN_TYPE=train"]
         self._configure(path, extra_cmake_defs=extra_defs)
         self._clean(self._base_path)
-        self._make(path)
+        self._build(path)
         self._install_benchmark(path)
         self._lit(path, True, False)
 
-    def _make(self, path):
-        make_cmd = self.opts.make
+    def _build(self, path):
+        cmake_cmd = self.opts.cmake
 
         subdir = path
         target = 'all'
@@ -576,24 +575,23 @@ def _make(self, path):
             subdir = os.path.join(*components)
 
         logger.info('Building...')
-        if not self.opts.succinct:
-            args = ["VERBOSE=1", target]
-        else:
-            args = [target]
         try:
-            self._check_call([make_cmd,
-                              '-k', '-j', str(self._build_threads())] + args,
+            self._check_call([cmake_cmd,
+                              '--build', '.',
+                              '-t', target,
+                              '-j', str(self._build_threads())] +
+                             ([] if self.opts.succinct else ["-v"]),
                              cwd=subdir)
         except subprocess.CalledProcessError:
-            # make is expected to exit with code 1 if there was any build
+            # cmake is expected to exit with code 1 if there was any build
             # failure. Build failures are not unexpected when testing an
             # experimental compiler.
             pass
 
     def _install_benchmark(self, path):
         if self.remote_run:
-            make_cmd = self.opts.make
-            self._check_call([make_cmd, 'rsync'], cwd=path)
+            cmake_cmd = self.opts.cmake
+            self._check_call([cmake_cmd, '--build', '.', '-t', 'rsync'], cwd=path)
 
     def _lit(self, path, test, profile):
         lit_cmd = self.opts.lit
@@ -898,30 +896,30 @@ def diagnose(self):
         logger.info(out)
 
         # Figure out our test's target.
-        make_cmd = [self.opts.make, "VERBOSE=1", 'help']
+        cmake_cmd = [self.opts.cmake, '--build', '.', '-t', 'help']
 
-        make_targets = subprocess.check_output(make_cmd,
-                                               universal_newlines=True)
+        cmake_targets = subprocess.check_output(cmake_cmd,
+                                                universal_newlines=True)
         matcher = re.compile(r"^\.\.\.\s{}$".format(short_name),
                              re.MULTILINE | re.IGNORECASE)
-        if not matcher.search(make_targets):
+        if not matcher.search(cmake_targets):
             assert False, "did not find benchmark, nestsed? Unimplemented."
 
         local_path = os.path.join(path, bm_path)
 
-        make_deps = [self.opts.make, "VERBOSE=1", "timeit-target",
-                     "timeit-host", "fpcmp-host"]
-        logger.info(" ".join(make_deps))
-        p = subprocess.Popen(make_deps,
+        cmake_deps = [self.opts.cmake, "--build", '.', "-t", "timeit-target",
+                      "timeit-host", "fpcmp-host"]
+        logger.info(" ".join(cmake_deps))
+        p = subprocess.Popen(cmake_deps,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT,
                              universal_newlines=True)
         std_out, std_err = p.communicate()
         logger.info(std_out)
 
-        make_save_temps = [self.opts.make, "VERBOSE=1", short_name]
-        logger.info(" ".join(make_save_temps))
-        p = subprocess.Popen(make_save_temps,
+        cmake_save_temps = [self.opts.cmake, "--build", '.', "-t", short_name]
+        logger.info(" ".join(cmake_save_temps))
+        p = subprocess.Popen(cmake_save_temps,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT,
                              universal_newlines=True)
@@ -951,8 +949,8 @@ def diagnose(self):
         out = subprocess.check_output(cmd_time_report, universal_newlines=True)
         logger.info(out)
 
-        make_time_report = [self.opts.make, "VERBOSE=1", short_name]
-        p = subprocess.Popen(make_time_report,
+        cmake_time_report = [self.opts.cmake, "--build", '.', "-t", short_name]
+        p = subprocess.Popen(cmake_time_report,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
                              universal_newlines=True)
@@ -971,8 +969,8 @@ def diagnose(self):
                                       universal_newlines=True)
         logger.info(out)
 
-        make_stats_report = [self.opts.make, "VERBOSE=1", short_name]
-        p = subprocess.Popen(make_stats_report,
+        cmake_stats_report = [self.opts.cmake, "--build", '.', "-t", short_name]
+        p = subprocess.Popen(cmake_stats_report,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
                              universal_newlines=True)
@@ -1002,8 +1000,8 @@ def diagnose(self):
             subprocess.check_output(cmd_iprofiler, universal_newlines=True)
 
             os.chdir(local_path)
-            make_iprofiler_temps = [self.opts.make, "VERBOSE=1", short_name]
-            p = subprocess.Popen(make_iprofiler_temps,
+            cmake_iprofiler_temps = [self.opts.cmake, "--build", '.', "-t", short_name]
+            p = subprocess.Popen(cmake_iprofiler_temps,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
             p.communicate()
@@ -1148,7 +1146,7 @@ def diagnose(self):
               help="write raw report data to PATH (or stdout if '-')",
               default=None)
 @click.option("--succinct-compile-output", "succinct",
-              help="run Make without VERBOSE=1", is_flag=True)
+              help="run CMake without -v", is_flag=True)
 @click.option("-v", "--verbose", "verbose", is_flag=True, default=False,
               help="show verbose test results")
 @click.option("--exclude-stat-from-submission",
@@ -1168,8 +1166,8 @@ def diagnose(self):
               type=click.UNPROCESSED, default="cmake",
               help="Path to CMake [cmake]")
 @click.option("--use-make", "make", metavar="PATH",
-              type=click.UNPROCESSED, default="make",
-              help="Path to Make [make]")
+              type=click.UNPROCESSED,
+              help="Path to the build system tool [make/ninja/...]")
 @click.option("--use-lit", "lit", metavar="PATH", type=click.UNPROCESSED,
               default="llvm-lit",
               help="Path to the LIT test runner [llvm-lit]")

tests/runtest/Inputs/test-suite-cmake/fake-cmake

@@ -23,6 +23,9 @@ do
     DUMP_VARS=true
     shift
     shift
+  elif [[ $1 == --build ]]; then
+    source $(dirname "${BASH_SOURCE[0]}")/fake-make
+    exit $?
   elif [[ ! -f $1/CMakeLists.txt && ! -f $1/CMakeCache.txt ]]; then
     # arguments not starting with -D or -C are assumed to point to the cmake
     # src or build dir

tests/runtest/Inputs/test-suite-cmake/fake-diagnose-cmake

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+if [[ $1 == --build && $2 == . && $3 == -t ]]
+then
+    if [ "$4" == "help" ]
+    then
+        echo "The following are some of the valid targets for this Makefile:
+... all (the default if no target is provided)
+... short_circuit_dtor
+... pointer_method2
+... pointer_member
+... 2003-08-20-EnumSizeProblem
+... BuiltinTypeInfo
+... ofstream_ctor
+... 2003-05-14-expr_stmt
+... 2011-03-28-Bitfield
+... 2008-01-29-ParamAliasesReturn
+... pointer_method
+... global_ctor
+... 2003-06-08-VirtualFunctions
+... Shootout-C++-hash
+... Bubblesort
+... lists
+"
+    elif [ "$4" == "Bubblesort" ]
+    then
+        #  Look like we ran the test and made outputs
+        mkdir -p SingleSource/Benchmarks/Stanford/CMakeFiles/Bubblesort.dir/
+        mkdir -p SingleSource/Benchmarks/Stanford/Bubblesort.dtps
+        touch SingleSource/Benchmarks/Stanford/Bubblesort.dtps/data
+        touch SingleSource/Benchmarks/Stanford/${4}
+        echo "RUN: Bubblesort | grep test" > SingleSource/Benchmarks/Stanford/${4}.test
+        echo "output file" > SingleSource/Benchmarks/Stanford/${4}.s
+        echo "output file" > SingleSource/Benchmarks/Stanford/${4}.i
+        echo "output file" > SingleSource/Benchmarks/Stanford/${4}.bc
+        echo "output file" > SingleSource/Benchmarks/Stanford/CMakeFiles/Bubblesort.dir/${4}.o
+        (>&2 echo "ftime-report data")
+    else
+        echo "Unexpected Argument"
+        exit 1
+    fi
+else
+    source $(dirname "${BASH_SOURCE[0]}")/fake-cmake "$@"
+    exit $?
+fi

tests/runtest/Inputs/test-suite-cmake/fake-diagnose-make

@@ -17,4 +17,4 @@
 # CHECK-ONLYTEST: Configuring with {
 # CHECK-ONLYTEST:   one: 'two'
 # CHECK-ONLYTEST:   three: 'four'
-# CHECK-ONLYTEST: Execute: {{.*}}/fake-make -k -j 1 VERBOSE=1 subtest
+# CHECK-ONLYTEST: Execute: {{.*}}/fake-cmake --build . -t subtest -j 1 -v

tests/runtest/test_suite-only-test.shtest

@@ -32,20 +32,20 @@
 # RUN:     > %t.pgo_multi.log 2> %t.pgo_multi.err
 # RUN: FileCheck --check-prefix CHECK-PGO-MULTI < %t.pgo_multi.err %s
 # CHECK-PGO-MULTI: TEST_SUITE_PROFILE_GENERATE: 'On'
-# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-make{{.*}} all
+# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-cmake --build {{.*}} -t all
 # CHECK-PGO-MULTI: fake-lit-profile
 # CHECK-PGO-MULTI: TEST_SUITE_PROFILE_GENERATE: 'Off'
 # CHECK-PGO-MULTI: TEST_SUITE_PROFILE_USE: 'On'
-# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-make{{.*}} clean
-# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-make{{.*}} all
+# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-cmake --build {{.*}} -t clean
+# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-cmake --build {{.*}} -t all
 # CHECK-PGO-MULTI: fake-lit-profile
 # CHECK-PGO-MULTI: TEST_SUITE_PROFILE_GENERATE: 'On'
 # CHECK-PGO-MULTI: TEST_SUITE_PROFILE_USE: 'Off'
-# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-make{{.*}} clean
-# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-make{{.*}} all
+# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-cmake --build {{.*}} -t clean
+# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-cmake --build {{.*}} -t all
 # CHECK-PGO-MULTI: fake-lit-profile
 # CHECK-PGO-MULTI: TEST_SUITE_PROFILE_GENERATE: 'Off'
 # CHECK-PGO-MULTI: TEST_SUITE_PROFILE_USE: 'On'
-# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-make{{.*}} clean
-# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-make{{.*}} all
+# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-cmake --build {{.*}} -t clean
+# CHECK-PGO-MULTI: Execute: {{.*}}/Inputs/test-suite-cmake/fake-cmake --build {{.*}} -t all
 # CHECK-PGO-MULTI: fake-lit-profile

tests/runtest/test_suite-pgo.shtest

@@ -24,8 +24,7 @@
 # RUN:     --no-timestamp \
 # RUN:     --test-suite %S/Inputs/test-suite-cmake \
 # RUN:     --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
-# RUN:     --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
-# RUN:     --use-make %S/Inputs/test-suite-cmake/fake-diagnose-make \
+# RUN:     --use-cmake %S/Inputs/test-suite-cmake/fake-diagnose-cmake \
 # RUN:     --use-lit %S/Inputs/test-suite-cmake/fake-lit \
 # RUN:     --diagnose --only-test SingleSource/Benchmarks/Stanford/Bubblesort \
 # RUN:     2>&1 | tee %t.diagnose.log