auto merge of #8458 : cmr/rust/test-restructure, r=brson

This should make benchmarks easier to understand. But, it doesn't work.
BENCH_RS in mk/tests.mk has everything, from what I can tell in remake, but
only those that are direct children of src/test/bench get build and run.
@graydon, can you lend your expertise? I can't make heads or tails of this
makefile.

Author: bors

mk/tests.mk

@@ -476,15 +476,15 @@ RFAIL_RC := $(wildcard $(S)src/test/run-fail/*.rc)
 RFAIL_RS := $(wildcard $(S)src/test/run-fail/*.rs)
 CFAIL_RC := $(wildcard $(S)src/test/compile-fail/*.rc)
 CFAIL_RS := $(wildcard $(S)src/test/compile-fail/*.rs)
-BENCH_RS := $(wildcard $(S)src/test/bench/*.rs)
+BENCH_RS := $(wildcard $(S)src/test/bench/rt/*.rs $(S)src/test/bench/shootout/*.rs $(S)src/test/bench/std/*.rs $(S)src/test/bench/*.rs)
 PRETTY_RS := $(wildcard $(S)src/test/pretty/*.rs)
 DEBUGINFO_RS := $(wildcard $(S)src/test/debug-info/*.rs)
 CODEGEN_RS := $(wildcard $(S)src/test/codegen/*.rs)
 CODEGEN_CC := $(wildcard $(S)src/test/codegen/*.cc)
 
 # perf tests are the same as bench tests only they run under
 # a performance monitor.
-PERF_RS := $(wildcard $(S)src/test/bench/*.rs)
+PERF_RS := $(BENCH_RS)
 
 RPASS_TESTS := $(RPASS_RC) $(RPASS_RS)
 RPASS_FULL_TESTS := $(RPASS_FULL_RC) $(RPASS_FULL_RS)
@@ -516,7 +516,7 @@ CTEST_BUILD_BASE_cfail = compile-fail
 CTEST_MODE_cfail = compile-fail
 CTEST_RUNTOOL_cfail = $(CTEST_RUNTOOL)
 
-CTEST_SRC_BASE_bench = bench
+CTEST_SRC_BASE_bench = bench bench/rt bench/shootout bench/std
 CTEST_BUILD_BASE_bench = bench
 CTEST_MODE_bench = run-pass
 CTEST_RUNTOOL_bench = $(CTEST_RUNTOOL)
@@ -610,7 +610,8 @@ define DEF_RUN_COMPILETEST
 
 CTEST_ARGS$(1)-T-$(2)-H-$(3)-$(4) := \
         $$(CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3))	\
-        --src-base $$(S)src/test/$$(CTEST_SRC_BASE_$(4))/ \
+        $(foreach base,$$(CTEST_SRC_BASE_$(4)), \
+        --src-base $$(S)src/test/$$(base))/ \
         --build-base $(3)/test/$$(CTEST_BUILD_BASE_$(4))/ \
         --ratchet-metrics $(call TEST_RATCHET_FILE,$(1),$(2),$(3),$(4)) \
         --mode $$(CTEST_MODE_$(4)) \

src/compiletest/common.rs

@@ -36,7 +36,7 @@ pub struct config {
     llvm_bin_path: Option<Path>,
 
     // The directory containing the tests to run
-    src_base: Path,
+    src_base: ~[Path],
 
     // The directory where programs should be built
     build_base: Path,

src/compiletest/compiletest.rs

@@ -19,7 +19,7 @@ use std::os;
 use std::f64;
 
 use extra::getopts;
-use extra::getopts::groups::{optopt, optflag, reqopt};
+use extra::getopts::groups::{optopt, optflag, reqopt, optmulti};
 use extra::test;
 
 use common::config;
@@ -49,19 +49,19 @@ pub fn main() {
 pub fn parse_config(args: ~[~str]) -> config {
 
     let groups : ~[getopts::groups::OptGroup] =
-        ~[reqopt("", "compile-lib-path", "path to host shared libraries", "PATH"),
-          reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
-          reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
-          optopt("", "clang-path", "path to  executable for codegen tests", "PATH"),
-          optopt("", "llvm-bin-path", "path to directory holding llvm binaries", "DIR"),
-          reqopt("", "src-base", "directory to scan for test files", "PATH"),
-          reqopt("", "build-base", "directory to deposit test outputs", "PATH"),
-          reqopt("", "aux-base", "directory to find auxiliary test files", "PATH"),
-          reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
-          reqopt("", "mode", "which sort of compile tests to run",
-                 "(compile-fail|run-fail|run-pass|pretty|debug-info)"),
-          optflag("", "ignored", "run tests marked as ignored / xfailed"),
-          optopt("", "runtool", "supervisor program to run tests under \
+        ~[reqopt   ("", "compile-lib-path", "path to host shared libraries", "PATH"),
+          reqopt   ("", "run-lib-path", "path to target shared libraries", "PATH"),
+          reqopt   ("", "rustc-path", "path to rustc to use for compiling", "PATH"),
+          optopt   ("", "clang-path", "path to  executable for codegen tests", "PATH"),
+          optopt   ("", "llvm-bin-path", "path to directory holding llvm binaries", "DIR"),
+          optmulti ("", "src-base", "directory to scan for test files", "PATH"),
+          reqopt   ("", "build-base", "directory to deposit test outputs", "PATH"),
+          reqopt   ("", "aux-base", "directory to find auxiliary test files", "PATH"),
+          reqopt   ("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
+          reqopt   ("", "mode", "which sort of compile tests to run",
+                    " (compile-fail|run-fail|run-pass|pretty|debug-info)"),
+          optflag  ("", "ignored", "run tests marked as ignored / xfailed"),
+          optopt   ("", "runtool", "supervisor program to run tests under \
                                  (eg. emulator, valgrind)", "PROGRAM"),
           optopt("", "rustcflags", "flags to pass to rustc", "FLAGS"),
           optflag("", "verbose", "run tests verbosely, showing all output"),
@@ -106,13 +106,15 @@ pub fn parse_config(args: ~[~str]) -> config {
         Path(getopts::opt_str(m, nm))
     }
 
+    let src_base = getopts::opt_strs(matches, "src-base");
+
     config {
         compile_lib_path: getopts::opt_str(matches, "compile-lib-path"),
         run_lib_path: getopts::opt_str(matches, "run-lib-path"),
         rustc_path: opt_path(matches, "rustc-path"),
         clang_path: getopts::opt_maybe_str(matches, "clang-path").map_move(|s| Path(s)),
         llvm_bin_path: getopts::opt_maybe_str(matches, "llvm-bin-path").map_move(|s| Path(s)),
-        src_base: opt_path(matches, "src-base"),
+        src_base: src_base.iter().map(|x| Path(x.clone())).collect(),
         build_base: opt_path(matches, "build-base"),
         aux_base: opt_path(matches, "aux-base"),
         stage_id: getopts::opt_str(matches, "stage-id"),
@@ -248,7 +250,7 @@ pub fn make_tests(config: &config) -> ~[test::TestDescAndFn] {
     debug!("making tests from %s",
            config.src_base.to_str());
     let mut tests = ~[];
-    let dirs = os::list_dir_path(&config.src_base);
+    let dirs = config.src_base.iter().flat_map(|x| os::list_dir_path(x).move_iter()).to_owned_vec();
     for file in dirs.iter() {
         let file = file.clone();
         debug!("inspecting file %s", file.to_str());