support creation of anonymous pipe on WinXP/2K3

Author: jsen-

src/libstd/sys/windows/c.rs

@@ -182,6 +182,7 @@ pub const ERROR_INVALID_HANDLE: DWORD = 6;
 pub const ERROR_NO_MORE_FILES: DWORD = 18;
 pub const ERROR_HANDLE_EOF: DWORD = 38;
 pub const ERROR_FILE_EXISTS: DWORD = 80;
+pub const ERROR_INVALID_PARAMETER: DWORD = 87;
 pub const ERROR_BROKEN_PIPE: DWORD = 109;
 pub const ERROR_CALL_NOT_IMPLEMENTED: DWORD = 120;
 pub const ERROR_INSUFFICIENT_BUFFER: DWORD = 122;

src/libstd/sys/windows/pipe.rs

@@ -43,6 +43,7 @@ pub fn anon_pipe() -> io::Result<(AnonPipe, AnonPipe)> {
         let reader;
         let mut name;
         let mut tries = 0;
+        let mut reject_remote_clients_flag = c::PIPE_REJECT_REMOTE_CLIENTS;
         loop {
             tries += 1;
             let key: u64 = rand::thread_rng().gen();
@@ -56,12 +57,12 @@ pub fn anon_pipe() -> io::Result<(AnonPipe, AnonPipe)> {
 
             let handle = c::CreateNamedPipeW(wide_name.as_ptr(),
                                              c::PIPE_ACCESS_INBOUND |
-                                              c::FILE_FLAG_FIRST_PIPE_INSTANCE |
-                                              c::FILE_FLAG_OVERLAPPED,
+                                             c::FILE_FLAG_FIRST_PIPE_INSTANCE |
+                                             c::FILE_FLAG_OVERLAPPED,
                                              c::PIPE_TYPE_BYTE |
-                                              c::PIPE_READMODE_BYTE |
-                                              c::PIPE_WAIT |
-                                              c::PIPE_REJECT_REMOTE_CLIENTS,
+                                             c::PIPE_READMODE_BYTE |
+                                             c::PIPE_WAIT |
+                                             reject_remote_clients_flag,
                                              1,
                                              4096,
                                              4096,
@@ -76,11 +77,27 @@ pub fn anon_pipe() -> io::Result<(AnonPipe, AnonPipe)> {
             //
             // Don't try again too much though as this could also perhaps be a
             // legit error.
+            // If ERROR_INVALID_PARAMETER is returned, this probably means we're
+            // running on pre-Vista version where PIPE_REJECT_REMOTE_CLIENTS is
+            // not supported, so we continue retrying without it. This implies
+            // reduced security on Windows versions older than Vista by allowing
+            // connections to this pipe from remote machines.
+            // Proper fix would increase the number of FFI imports and introduce
+            // significant amount of Windows XP specific code with no clean
+            // testing strategy
+            // for more info see https://github.com/rust-lang/rust/pull/37677
             if handle == c::INVALID_HANDLE_VALUE {
                 let err = io::Error::last_os_error();
-                if tries < 10 &&
-                   err.raw_os_error() == Some(c::ERROR_ACCESS_DENIED as i32) {
-                    continue
+                let raw_os_err = err.raw_os_error();
+                if tries < 10 {
+                    if raw_os_err == Some(c::ERROR_ACCESS_DENIED as i32) {
+                        continue
+                    } else if reject_remote_clients_flag != 0 &&
+                        raw_os_err == Some(c::ERROR_INVALID_PARAMETER as i32) {
+                        reject_remote_clients_flag = 0;
+                        tries -= 1;
+                        continue
+                    }
                 }
                 return Err(err)
             }