Merge pull request #416 from python-cmd2/pyshell_readline

Pyshell readline

Author: kmvanbrunt

CHANGELOG.md

@@ -2,6 +2,7 @@
 * Bug Fixes
     * If self.default_to_shell is true, then redirection and piping are now properly passed to the shell. Previously it was truncated.
     * Submenus now call all hooks, it used to just call precmd and postcmd.
+    * Fixed ``AttributeError`` on Windows when running a ``select`` command cause by **pyreadline** not implementing ``remove_history_item``
 * Enhancements
     * Automatic completion of ``argparse`` arguments via ``cmd2.argparse_completer.AutoCompleter``
         * See the [tab_autocompletion.py](https://github.com/python-cmd2/cmd2/blob/master/examples/tab_autocompletion.py) example for a demonstration of how to use this feature
@@ -16,6 +17,9 @@
         * ``identchars`` is now ignored. The standardlibrary cmd uses those characters to split the first "word" of the input, but cmd2 hasn't used those for a while, and the new parsing logic parses on whitespace, which has the added benefit of full unicode support, unlike cmd or prior versions of cmd2.
         * ``set_posix_shlex`` function and ``POSIX_SHLEX`` variable have been removed. Parsing behavior is now always the more forgiving ``posix=false``.
         * ``set_strip_quotes`` function and ``STRIP_QUOTES_FOR_NON_POSIX`` have been removed. Quotes are stripped from arguments when presented as a list (a la ``sys.argv``), and present when arguments are presented as a string (like the string passed to do_*).
+    * Enhanced the ``py`` console in the following ways
+        * Added tab completion of Python identifiers instead of **cmd2** commands
+        * Separated the ``py`` console history from the **cmd2** history
 * Changes
     * ``strip_ansi()`` and ``strip_quotes()`` functions have moved to new utils module
     * Several constants moved to new constants module

cmd2/cmd2.py

@@ -58,6 +58,9 @@
 else:
     from .rl_utils import rl_force_redisplay, readline
 
+    # Used by rlcompleter in Python console loaded by py command
+    orig_rl_delims = readline.get_completer_delims()
+
     if rl_type == RlType.PYREADLINE:
 
         # Save the original pyreadline display completion function since we need to override it and restore it
@@ -73,6 +76,9 @@
         import ctypes
         from .rl_utils import readline_lib
 
+        rl_basic_quote_characters = ctypes.c_char_p.in_dll(readline_lib, "rl_basic_quote_characters")
+        orig_rl_basic_quotes = ctypes.cast(rl_basic_quote_characters, ctypes.c_void_p).value
+
 from .argparse_completer import AutoCompleter, ACArgumentParser
 
 # Newer versions of pyperclip are released as a single file, but older versions had a more complicated structure
@@ -416,6 +422,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
         self.initial_stdout = sys.stdout
         self.history = History()
         self.pystate = {}
+        self.py_history = []
         self.pyscript_name = 'app'
         self.keywords = self.reserved_words + [fname[3:] for fname in dir(self) if fname.startswith('do_')]
         self.statement_parser = StatementParser(
@@ -476,7 +483,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
 
         ############################################################################################################
         # The following variables are used by tab-completion functions. They are reset each time complete() is run
-        # using set_completion_defaults() and it is up to completer functions to set them before returning results.
+        # in reset_completion_defaults() and it is up to completer functions to set them before returning results.
         ############################################################################################################
 
         # If true and a single match is returned to complete(), then a space will be appended
@@ -643,7 +650,7 @@ def colorize(self, val, color):
 
     # -----  Methods related to tab completion -----
 
-    def set_completion_defaults(self):
+    def reset_completion_defaults(self):
         """
         Resets tab completion settings
         Needs to be called each time readline runs tab completion
@@ -1285,7 +1292,7 @@ def complete(self, text, state):
         import functools
         if state == 0 and rl_type != RlType.NONE:
             unclosed_quote = ''
-            self.set_completion_defaults()
+            self.reset_completion_defaults()
 
             # lstrip the original line
             orig_line = readline.get_line_buffer()
@@ -2026,12 +2033,10 @@ def _cmdloop(self):
                 # Set GNU readline's rl_basic_quote_characters to NULL so it won't automatically add a closing quote
                 # We don't need to worry about setting rl_completion_suppress_quote since we never declared
                 # rl_completer_quote_characters.
-                basic_quote_characters = ctypes.c_char_p.in_dll(readline_lib, "rl_basic_quote_characters")
-                old_basic_quote_characters = ctypes.cast(basic_quote_characters, ctypes.c_void_p).value
-                basic_quote_characters.value = None
+                old_basic_quotes = ctypes.cast(rl_basic_quote_characters, ctypes.c_void_p).value
+                rl_basic_quote_characters.value = None
 
             old_completer = readline.get_completer()
-            old_delims = readline.get_completer_delims()
             readline.set_completer(self.complete)
 
             # Break words on whitespace and quotes when tab completing
@@ -2041,6 +2046,7 @@ def _cmdloop(self):
                 # If redirection is allowed, then break words on those characters too
                 completer_delims += ''.join(constants.REDIRECTION_CHARS)
 
+            old_delims = readline.get_completer_delims()
             readline.set_completer_delims(completer_delims)
 
             # Enable tab completion
@@ -2077,7 +2083,7 @@ def _cmdloop(self):
 
                 if rl_type == RlType.GNU:
                     readline.set_completion_display_matches_hook(None)
-                    basic_quote_characters.value = old_basic_quote_characters
+                    rl_basic_quote_characters.value = old_basic_quotes
                 elif rl_type == RlType.PYREADLINE:
                     readline.rl.mode._display_completions = orig_pyreadline_display
 
@@ -2507,7 +2513,30 @@ def complete_shell(self, text, line, begidx, endidx):
         index_dict = {1: self.shell_cmd_complete}
         return self.index_based_complete(text, line, begidx, endidx, index_dict, self.path_complete)
 
-    # noinspection PyBroadException
+    @staticmethod
+    def _reset_py_display() -> None:
+        """
+        Resets the dynamic objects in the sys module that the py and ipy consoles fight over.
+        When a Python console starts it adopts certain display settings if they've already been set.
+        If an ipy console has previously been run, then py uses its settings and ends up looking
+        like an ipy console in terms of prompt and exception text. This method forces the Python
+        console to create its own display settings since they won't exist.
+
+        IPython does not have this problem since it always overwrites the display settings when it
+        is run. Therefore this method only needs to be called before creating a Python console.
+        """
+        # Delete any prompts that have been set
+        attributes = ['ps1', 'ps2', 'ps3']
+        for cur_attr in attributes:
+            try:
+                del sys.__dict__[cur_attr]
+            except KeyError:
+                pass
+
+        # Reset functions
+        sys.displayhook = sys.__displayhook__
+        sys.excepthook = sys.__excepthook__
+
     def do_py(self, arg):
         """
         Invoke python command, shell, or script
@@ -2524,6 +2553,7 @@ def do_py(self, arg):
             return
         self._in_py = True
 
+        # noinspection PyBroadException
         try:
             arg = arg.strip()
 
@@ -2539,6 +2569,7 @@ def run(filename):
                 except IOError as e:
                     self.perror(e)
 
+            # noinspection PyUnusedLocal
             def onecmd_plus_hooks(cmd_plus_args):
                 """Run a cmd2.Cmd command from a Python script or the interactive Python console.
 
@@ -2561,6 +2592,8 @@ def onecmd_plus_hooks(cmd_plus_args):
 
             if arg:
                 interp.runcode(arg)
+
+            # If there are no args, then we will open an interactive Python console
             else:
                 # noinspection PyShadowingBuiltins
                 def quit():
@@ -2570,20 +2603,98 @@ def quit():
                 self.pystate['quit'] = quit
                 self.pystate['exit'] = quit
 
-                keepstate = None
+                # Set up readline for Python console
+                if rl_type != RlType.NONE:
+                    # Save cmd2 history
+                    saved_cmd2_history = []
+                    for i in range(1, readline.get_current_history_length() + 1):
+                        saved_cmd2_history.append(readline.get_history_item(i))
+
+                    readline.clear_history()
+
+                    # Restore py's history
+                    for item in self.py_history:
+                        readline.add_history(item)
+
+                    if self.use_rawinput and self.completekey:
+                        # Set up tab completion for the Python console
+                        # rlcompleter relies on the default settings of the Python readline module
+                        if rl_type == RlType.GNU:
+                            old_basic_quotes = ctypes.cast(rl_basic_quote_characters, ctypes.c_void_p).value
+                            rl_basic_quote_characters.value = orig_rl_basic_quotes
+
+                            if 'gnureadline' in sys.modules:
+                                # rlcompleter imports readline by name, so it won't use gnureadline
+                                # Force rlcompleter to use gnureadline instead so it has our settings and history
+                                saved_readline = None
+                                if 'readline' in sys.modules:
+                                    saved_readline = sys.modules['readline']
+
+                                sys.modules['readline'] = sys.modules['gnureadline']
+
+                        old_delims = readline.get_completer_delims()
+                        readline.set_completer_delims(orig_rl_delims)
+
+                        # rlcompleter will not need cmd2's custom display function
+                        # This will be restored by cmd2 the next time complete() is called
+                        if rl_type == RlType.GNU:
+                            readline.set_completion_display_matches_hook(None)
+                        elif rl_type == RlType.PYREADLINE:
+                            readline.rl.mode._display_completions = self._display_matches_pyreadline
+
+                        # Save off the current completer and set a new one in the Python console
+                        # Make sure it tab completes from its locals() dictionary
+                        old_completer = readline.get_completer()
+                        interp.runcode("from rlcompleter import Completer")
+                        interp.runcode("import readline")
+                        interp.runcode("readline.set_completer(Completer(locals()).complete)")
+
+                # Set up sys module for the Python console
+                self._reset_py_display()
+                keepstate = Statekeeper(sys, ('stdin', 'stdout'))
+                sys.stdout = self.stdout
+                sys.stdin = self.stdin
+
+                cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
+                docstr = self.do_py.__doc__.replace('pyscript_name', self.pyscript_name)
+
                 try:
-                    cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
-                    keepstate = Statekeeper(sys, ('stdin', 'stdout'))
-                    sys.stdout = self.stdout
-                    sys.stdin = self.stdin
-                    docstr = self.do_py.__doc__.replace('pyscript_name', self.pyscript_name)
-                    interp.interact(banner="Python %s on %s\n%s\n(%s)\n%s" %
-                                           (sys.version, sys.platform, cprt, self.__class__.__name__,
-                                            docstr))
+                    interp.interact(banner="Python {} on {}\n{}\n({})\n{}".
+                                    format(sys.version, sys.platform, cprt, self.__class__.__name__, docstr))
                 except EmbeddedConsoleExit:
                     pass
-                if keepstate is not None:
+
+                finally:
                     keepstate.restore()
+
+                    # Set up readline for cmd2
+                    if rl_type != RlType.NONE:
+                        # Save py's history
+                        self.py_history.clear()
+                        for i in range(1, readline.get_current_history_length() + 1):
+                            self.py_history.append(readline.get_history_item(i))
+
+                        readline.clear_history()
+
+                        # Restore cmd2's history
+                        for item in saved_cmd2_history:
+                            readline.add_history(item)
+
+                        if self.use_rawinput and self.completekey:
+                            # Restore cmd2's tab completion settings
+                            readline.set_completer(old_completer)
+                            readline.set_completer_delims(old_delims)
+
+                            if rl_type == RlType.GNU:
+                                rl_basic_quote_characters.value = old_basic_quotes
+
+                                if 'gnureadline' in sys.modules:
+                                    # Restore what the readline module pointed to
+                                    if saved_readline is None:
+                                        del(sys.modules['readline'])
+                                    else:
+                                        sys.modules['readline'] = saved_readline
+
         except Exception:
             pass
         finally:

cmd2/rl_utils.py

@@ -33,6 +33,38 @@ class RlType(Enum):
 if 'pyreadline' in sys.modules:
     rl_type = RlType.PYREADLINE
 
+    ############################################################################################################
+    # pyreadline is incomplete in terms of the Python readline API. Add the missing functions we need.
+    ############################################################################################################
+    # readline.redisplay()
+    try:
+        getattr(readline, 'redisplay')
+    except AttributeError:
+        # noinspection PyProtectedMember
+        readline.redisplay = readline.rl.mode._update_line
+
+    # readline.remove_history_item()
+    try:
+        getattr(readline, 'remove_history_item')
+    except AttributeError:
+        # noinspection PyProtectedMember
+        def pyreadline_remove_history_item(pos: int) -> None:
+            """
+            An implementation of remove_history_item() for pyreadline
+            :param pos: The 0-based position in history to remove
+            """
+            # Save of the current location of the history cursor
+            saved_cursor = readline.rl.mode._history.history_cursor
+
+            # Delete the history item
+            del(readline.rl.mode._history.history[pos])
+
+            # Update the cursor if needed
+            if saved_cursor > pos:
+                readline.rl.mode._history.history_cursor -= 1
+
+        readline.remove_history_item = pyreadline_remove_history_item
+
 elif 'gnureadline' in sys.modules or 'readline' in sys.modules:
     # We don't support libedit
     if 'libedit' not in readline.__doc__: