Updated documentation. Removed sleep to speed up parallel connects. Updated tests.

Author: Dan

pssh/pssh_client.py

@@ -103,7 +103,7 @@ def __init__(self, hosts,
         >>> ... pass
         
         >>> # Commands have started executing at this point
-        >>> # Exit code will probably not be available immediately
+        >>> # Exit code will not be available immediately
         >>> print output
 
         ::
@@ -135,10 +135,24 @@ def __init__(self, hosts,
         [myhost1]     drwxrwxr-x 6 user group 4.0K Jan 1 HH:MM x
         [myhost2]     drwxrwxr-x 6 user group 4.0K Jan 1 HH:MM x
         
-        Retrieve exit codes after commands have finished as below. This is
-        only necessary for long running commands that do not exit immediately.
+        Retrieve exit codes after commands have finished as below.
         
-        ``exit_code`` in ``output`` will be ``None`` if command has not finished.
+        `parallel-ssh` starts commands asynchronously to enable running multiple
+        commands in parallel without blocking.
+
+        Because of this, exit codes will not be immediately available even for
+        commands that exit immediately.
+        
+        At least one of ::
+        
+        * Iterating over stdout/stderr
+        * Calling `client.join(output)`
+        * Calling `client.pool.join()` if no output is needed
+        
+        is necessary to cause `parallel-ssh` to wait for commands to finish and
+        be able to gather exit codes.
+        
+        ``exit_code`` in ``output`` will be ``None`` if command has not yet finished.
         
         ``get_exit_codes`` is not a blocking function and will not wait for commands
         to finish. Use ``client.join(output)`` to block until all commands have
@@ -172,6 +186,12 @@ def __init__(self, hosts,
         >>> hosts = ['dc1.myhost1', 'dc2.myhost2']
         >>> client = ParallelSSHClient([h for h in hosts if h.find('dc1')])
         >>> client.run_command(<..>)
+
+        **Overriding host list**
+
+        >>> client.hosts = ['otherhost']
+        >>> print client.run_command('exit 0')
+        >>> {'otherhost': {'exit_code':0}, <..>}
         
         .. note ::
         
@@ -276,6 +296,34 @@ def run_command(self, *args, **kwargs):
         >>> for host in output:
         >>>     stdout = list(output[host]['stdout'])
         >>>     print "Complete stdout for host %s is %s" % (host, stdout,)
+
+        **Run multiple commands in parallel**
+
+        This short example demonstrates running long running commands in parallel
+        and how long it takes for all commands to start, blocking until they
+        complete and how long it takes for all commands to complete.
+        
+        See examples directory for complete example script. ::
+        
+        output = []
+        
+        start = datetime.datetime.now()
+        cmds = ['sleep 5' for _ in xrange(10)]
+        for cmd in cmds:
+            output.append(client.run_command(cmd, stop_on_errors=False))
+        end = datetime.datetime.now()
+        print "Started %s commands in %s" % (len(cmds), end-start,)
+        start = datetime.datetime.now()
+        for _output in output:
+            for line in _output[host]['stdout']:
+                print line
+        end = datetime.datetime.now()
+        print "All commands finished in %s" % (end-start,)
+        
+        *Output*
+        
+        Started 10 commands in 0:00:00.428629
+        All commands finished in 0:00:05.014757
         
         **Example Output**
         

pssh/ssh_client.py

@@ -139,12 +139,14 @@ def _connect_tunnel(self):
           proxy_channel = self.proxy_client.get_transport().\
             open_channel('direct-tcpip', (self.host, self.port,),
                         ('127.0.0.1', 0))
+          gevent.sleep(0)
           return self._connect(self.client, self.host, self.port, sock=proxy_channel)
         except channel_exception, ex:
           error_type = ex.args[1] if len(ex.args) > 1 else ex.args[0]
           raise ConnectionErrorException("Error connecting to host '%s:%s' - %s",
                                            self.host, self.port,
                                            str(error_type))
+    
     def _connect(self, client, host, port, sock=None, retries=1):
         """Connect to host
         
@@ -229,7 +231,7 @@ def exec_command(self, command, sudo=False, user=None, **kwargs):
         logger.debug("Running command %s on %s", command, self.host)
         channel.exec_command(command, **kwargs)
         logger.debug("Command started")
-        gevent.sleep(.2)
+        gevent.sleep(0)
         return channel, self.host, stdout, stderr
 
     def _read_output_buffer(self, output_buffer, prefix=''):

tests/test_pssh_client.py

@@ -73,6 +73,7 @@ def test_pssh_client_exec_command(self):
     def test_pssh_client_no_stdout_non_zero_exit_code_immediate_exit(self):
         output = self.client.run_command('exit 1')
         expected_exit_code = 1
+        self.client.join(output)
         exit_code = output[self.host]['exit_code']
         self.assertEqual(expected_exit_code, exit_code,
                          msg="Got unexpected exit code - %s, expected %s" %
@@ -112,9 +113,9 @@ def test_pssh_client_run_command_get_output(self):
         expected_exit_code = 0
         expected_stdout = [self.fake_resp]
         expected_stderr = []
-        exit_code = output[self.host]['exit_code']
         stdout = list(output[self.host]['stdout'])
         stderr = list(output[self.host]['stderr'])
+        exit_code = output[self.host]['exit_code']
         self.assertEqual(expected_exit_code, exit_code,
                          msg="Got unexpected exit code - %s, expected %s" %
                          (exit_code,
@@ -439,6 +440,7 @@ def test_ssh_proxy(self):
                                    proxy_host='127.0.0.2',
                                    proxy_port=proxy_server_port
                                    )
+        gevent.sleep(2)
         output = client.run_command(self.fake_cmd)
         stdout = list(output[self.host]['stdout'])
         expected_stdout = [self.fake_resp]