auto merge of #8645 : alexcrichton/rust/issue-6436-run-non-blocking, r=brson

This overhauls `std::run` to instead run on top of libuv. This is *not* in a mergeable state, I've been attempting to diagnose failures in the compiletest suite. I've managed to find a fair number of bugs so far, but I still  don't seem to be done yet.

Notable changes:
* This requires upgrading libuv. From the discussion on #6567, I took libuv master from a few days ago, applied one patch to fix process spawning with multiple event loops in libuv, and pushed to my own fork
* The build system for libuv has changed since we last used it. There's some extra checkout from a google build system which apparently does all the magic if you don't want to require autotools, and the google system just requires python. I updated the Makefile to get this build system and build libuv with it instead. This is untested on windows and arm, and both will probably need to see some improvement.
* This required adding some pipe bindings to libuv as well. Currently the support is pretty simple and probably completely unsafe for pipes, but you at least get read/write methods. This is necessary for capturing output of processes.
* I didn't redesign `std::run` at all, I simply tried to reimplement all the existing functionality on top of libuv. Some functions ended up dying, but nothing major. All uses of `std::run` in the compiler still work just fine.

I'm not quite sure how the rest of the runtime deals with this, but I marked process structures as `no_send` because the waiting/waking up has to happen in the same event loop right now. If processes start migrating between event loops then very bad things can happen. This may be what threadsafe I/O would fix, and I would be more than willing to rebase on that if it lands first.

Anyway, for now I wanted to put this up for review, I'm still investigating the corruption/deadlock bugs, but this is in an *almost* workable state. Once I find the bugs I'll also rebase on the current master.

Author: bors

.gitmodules

@@ -4,5 +4,5 @@
 	branch = master
 [submodule "src/libuv"]
 	path = src/libuv
-	url = https://github.com/brson/libuv.git
+	url = https://github.com/alexcrichton/libuv.git
 	branch = master

mk/rt.mk

@@ -163,34 +163,49 @@ LIBUV_DEPS := $$(wildcard \
               $$(S)src/libuv/*/*/*/*)
 endif
 
+LIBUV_GYP := $$(S)src/libuv/build/gyp
+LIBUV_MAKEFILE_$(1)_$(2) := $$(CFG_BUILD_DIR)rt/$(1)/stage$(2)/libuv/Makefile
+LIBUV_NO_LOAD = run-benchmarks.target.mk run-tests.target.mk \
+		uv_dtrace_header.target.mk uv_dtrace_provider.target.mk
+
+$$(LIBUV_MAKEFILE_$(1)_$(2)): $$(LIBUV_GYP)
+	(cd $(S)src/libuv/ && \
+	 ./gyp_uv -f make -Dtarget_arch=$$(HOST_$(1)) -D ninja \
+	   -Goutput_dir=$$(@D) --generator-output $$(@D))
+
 # XXX: Shouldn't need platform-specific conditions here
 ifdef CFG_WINDOWSY_$(1)
 $$(LIBUV_LIB_$(1)_$(2)): $$(LIBUV_DEPS)
-	$$(Q)$$(MAKE) -C $$(S)src/libuv/ \
-		builddir_name="$$(CFG_BUILD_DIR)/rt/$(1)/stage$(2)/libuv" \
-		OS=mingw \
+	$$(Q)rm -f $$(S)src/libuv/libuv.a
+	$$(Q)$$(MAKE) -C $$(S)src/libuv -f Makefile.mingw \
+		CFLAGS="$$(CFG_GCCISH_CFLAGS) $$(LIBUV_FLAGS_$$(HOST_$(1))) $$(SNAP_DEFINES)" \
+		AR="$$(AR_$(1))" \
 		V=$$(VERBOSE)
+	$$(Q)cp $$(S)src/libuv/libuv.a $$@
 else ifeq ($(OSTYPE_$(1)), linux-androideabi)
-$$(LIBUV_LIB_$(1)_$(2)): $$(LIBUV_DEPS)
-	$$(Q)$$(MAKE) -C $$(S)src/libuv/ \
+$$(LIBUV_LIB_$(1)_$(2)): $$(LIBUV_DEPS) $$(LIBUV_MAKEFILE_$(1)_$(2))
+	$$(Q)$$(MAKE) -C $$(@D) \
 		CFLAGS="$$(CFG_GCCISH_CFLAGS) $$(LIBUV_FLAGS_$$(HOST_$(1))) $$(SNAP_DEFINES)" \
 		LDFLAGS="$$(CFG_GCCISH_LINK_FLAGS) $$(LIBUV_FLAGS_$$(HOST_$(1)))" \
 		CC="$$(CC_$(1))" \
 		CXX="$$(CXX_$(1))" \
 		AR="$$(AR_$(1))" \
-		BUILDTYPE=Release \
-		builddir_name="$$(CFG_BUILD_DIR)/rt/$(1)/stage$(2)/libuv" \
 		host=android OS=linux \
+		builddir="." \
+		BUILDTYPE=Release \
+		NO_LOAD="$$(LIBUV_NO_LOAD)" \
 		V=$$(VERBOSE)
 else
-$$(LIBUV_LIB_$(1)_$(2)): $$(LIBUV_DEPS)
-	$$(Q)$$(MAKE) -C $$(S)src/libuv/ \
+$$(LIBUV_LIB_$(1)_$(2)): $$(LIBUV_DEPS) $$(LIBUV_MAKEFILE_$(1)_$(2))
+	$$(Q)$$(MAKE) -C $$(@D) \
 		CFLAGS="$$(CFG_GCCISH_CFLAGS) $$(LIBUV_FLAGS_$$(HOST_$(1))) $$(SNAP_DEFINES)" \
 		LDFLAGS="$$(CFG_GCCISH_LINK_FLAGS) $$(LIBUV_FLAGS_$$(HOST_$(1)))" \
 		CC="$$(CC_$(1))" \
 		CXX="$$(CXX_$(1))" \
 		AR="$$(AR_$(1))" \
-		builddir_name="$$(CFG_BUILD_DIR)/rt/$(1)/stage$(2)/libuv" \
+		builddir="." \
+		BUILDTYPE=Release \
+		NO_LOAD="$$(LIBUV_NO_LOAD)" \
 		V=$$(VERBOSE)
 endif
 
@@ -254,3 +269,7 @@ endef
 $(foreach stage,$(STAGES), \
 	$(foreach target,$(CFG_TARGET_TRIPLES), \
 	 $(eval $(call DEF_RUNTIME_TARGETS,$(target),$(stage)))))
+
+$(LIBUV_GYP):
+	mkdir -p $(S)src/libuv/build
+	git clone https://git.chromium.org/external/gyp.git $(S)src/libuv/build/gyp

src/compiletest/procsrv.rs

@@ -54,10 +54,10 @@ pub fn run(lib_path: &str,
         in_fd: None,
         out_fd: None,
         err_fd: None
-    });
+    }).unwrap();
 
     for input in input.iter() {
-        proc.input().write_str(*input);
+        proc.input().write(input.as_bytes());
     }
     let output = proc.finish_with_output();
 

src/compiletest/runtest.rs

@@ -20,41 +20,16 @@ use procsrv;
 use util;
 use util::logv;
 
-use std::cell::Cell;
 use std::io;
 use std::os;
 use std::str;
-use std::task::{spawn_sched, SingleThreaded};
 use std::vec;
-use std::unstable::running_on_valgrind;
 
 use extra::test::MetricMap;
 
 pub fn run(config: config, testfile: ~str) {
-    let config = Cell::new(config);
-    let testfile = Cell::new(testfile);
-    // FIXME #6436: Creating another thread to run the test because this
-    // is going to call waitpid. The new scheduler has some strange
-    // interaction between the blocking tasks and 'friend' schedulers
-    // that destroys parallelism if we let normal schedulers block.
-    // It should be possible to remove this spawn once std::run is
-    // rewritten to be non-blocking.
-    //
-    // We do _not_ create another thread if we're running on V because
-    // it serializes all threads anyways.
-    if running_on_valgrind() {
-        let config = config.take();
-        let testfile = testfile.take();
-        let mut _mm = MetricMap::new();
-        run_metrics(config, testfile, &mut _mm);
-    } else {
-        do spawn_sched(SingleThreaded) {
-            let config = config.take();
-            let testfile = testfile.take();
-            let mut _mm = MetricMap::new();
-            run_metrics(config, testfile, &mut _mm);
-        }
-    }
+    let mut _mm = MetricMap::new();
+    run_metrics(config, testfile, &mut _mm);
 }
 
 pub fn run_metrics(config: config, testfile: ~str, mm: &mut MetricMap) {

src/librustdoc/markdown_writer.rs

@@ -104,14 +104,14 @@ fn pandoc_writer(
     ];
 
     do generic_writer |markdown| {
-        use std::io::WriterUtil;
-
         debug!("pandoc cmd: %s", pandoc_cmd);
         debug!("pandoc args: %s", pandoc_args.connect(" "));
 
-        let mut proc = run::Process::new(pandoc_cmd, pandoc_args, run::ProcessOptions::new());
+        let proc = run::Process::new(pandoc_cmd, pandoc_args,
+                                     run::ProcessOptions::new());
+        let mut proc = proc.unwrap();
 
-        proc.input().write_str(markdown);
+        proc.input().write(markdown.as_bytes());
         let output = proc.finish_with_output();
 
         debug!("pandoc result: %i", output.status);

src/librustpkg/source_control.rs

@@ -89,7 +89,7 @@ pub fn git_clone_general(source: &str, target: &Path, v: &Version) -> bool {
 
 fn process_output_in_cwd(prog: &str, args: &[~str], cwd: &Path) -> ProcessOutput {
     let mut prog = Process::new(prog, args, ProcessOptions{ dir: Some(cwd)
-                                ,..ProcessOptions::new()});
+                                ,..ProcessOptions::new()}).unwrap();
     prog.finish_with_output()
 }
 

src/librustpkg/tests.rs

@@ -112,13 +112,14 @@ fn mk_temp_workspace(short_name: &Path, version: &Version) -> Path {
 
 fn run_git(args: &[~str], env: Option<~[(~str, ~str)]>, cwd: &Path, err_msg: &str) {
     let cwd = (*cwd).clone();
-    let mut prog = run::Process::new("git", args, run::ProcessOptions {
+    let prog = run::Process::new("git", args, run::ProcessOptions {
         env: env,
         dir: Some(&cwd),
         in_fd: None,
         out_fd: None,
         err_fd: None
     });
+    let mut prog = prog.unwrap();
     let rslt = prog.finish_with_output();
     if rslt.status != 0 {
         fail!("%s [git returned %?, output = %s, error = %s]", err_msg,
@@ -226,7 +227,7 @@ fn command_line_test_with_env(args: &[~str], cwd: &Path, env: Option<~[(~str, ~s
         in_fd: None,
         out_fd: None,
         err_fd: None
-    });
+    }).unwrap();
     let output = prog.finish_with_output();
     debug!("Output from command %s with args %? was %s {%s}[%?]",
                     cmd, args, str::from_bytes(output.output),
@@ -1027,16 +1028,17 @@ fn test_extern_mod() {
                      test_sysroot().to_str(),
                      exec_file.to_str());
 
-    let mut prog = run::Process::new(rustc.to_str(), [main_file.to_str(),
-                                                      ~"--sysroot", test_sysroot().to_str(),
-                                               ~"-o", exec_file.to_str()],
-                                     run::ProcessOptions {
+    let prog = run::Process::new(rustc.to_str(), [main_file.to_str(),
+                                                  ~"--sysroot", test_sysroot().to_str(),
+                                                  ~"-o", exec_file.to_str()],
+                                 run::ProcessOptions {
         env: env,
         dir: Some(&dir),
         in_fd: None,
         out_fd: None,
         err_fd: None
     });
+    let mut prog = prog.unwrap();
     let outp = prog.finish_with_output();
     if outp.status != 0 {
         fail!("output was %s, error was %s",

src/libstd/rt/io/file.rs

@@ -71,9 +71,6 @@ pub struct FileStream {
     last_nread: int,
 }
 
-impl FileStream {
-}
-
 impl Reader for FileStream {
     fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
         match self.fd.read(buf) {

src/libstd/rt/io/mod.rs

@@ -268,6 +268,9 @@ pub use self::extensions::WriterByteConversions;
 /// Synchronous, non-blocking file I/O.
 pub mod file;
 
+/// Synchronous, in-memory I/O.
+pub mod pipe;
+
 /// Synchronous, non-blocking network I/O.
 pub mod net {
     pub mod tcp;

src/libstd/rt/io/net/tcp.rs

@@ -16,7 +16,7 @@ use rt::io::{io_error, read_error, EndOfFile};
 use rt::rtio::{IoFactory, IoFactoryObject,
                RtioSocket, RtioTcpListener,
                RtioTcpListenerObject, RtioTcpStream,
-               RtioTcpStreamObject};
+               RtioTcpStreamObject, RtioStream};
 use rt::local::Local;
 
 pub struct TcpStream(~RtioTcpStreamObject);
@@ -69,7 +69,7 @@ impl TcpStream {
 
 impl Reader for TcpStream {
     fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
-        match (**self).read(buf) {
+        match (***self).read(buf) {
             Ok(read) => Some(read),
             Err(ioerr) => {
                 // EOF is indicated by returning None
@@ -86,7 +86,7 @@ impl Reader for TcpStream {
 
 impl Writer for TcpStream {
     fn write(&mut self, buf: &[u8]) {
-        match (**self).write(buf) {
+        match (***self).write(buf) {
             Ok(_) => (),
             Err(ioerr) => io_error::cond.raise(ioerr),
         }
@@ -166,7 +166,7 @@ mod test {
         do run_in_newsched_task {
             let mut called = false;
             do io_error::cond.trap(|e| {
-                assert!(e.kind == ConnectionRefused);
+                assert_eq!(e.kind, ConnectionRefused);
                 called = true;
             }).inside {
                 let addr = SocketAddr { ip: Ipv4Addr(0, 0, 0, 0), port: 1 };