bpo-37903: IDLE: Shell sidebar with prompts (pythonGH-22682)

The first followup will change shell indents to spaces.
More are expected.

Co-authored-by: Terry Jan Reedy <[email protected]>

Author: taleinat

Lib/idlelib/colorizer.py

@@ -133,7 +133,6 @@ def LoadTagDefs(self):
             # non-modal alternative.
             "hit": idleConf.GetHighlight(theme, "hit"),
             }
-
         if DEBUG: print('tagdefs', self.tagdefs)
 
     def insert(self, index, chars, tags=None):

Lib/idlelib/editor.py

@@ -60,14 +60,14 @@ class EditorWindow:
     from idlelib.sidebar import LineNumbers
     from idlelib.format import FormatParagraph, FormatRegion, Indents, Rstrip
     from idlelib.parenmatch import ParenMatch
-    from idlelib.squeezer import Squeezer
     from idlelib.zoomheight import ZoomHeight
 
     filesystemencoding = sys.getfilesystemencoding()  # for file names
     help_url = None
 
     allow_code_context = True
     allow_line_numbers = True
+    user_input_insert_tags = None
 
     def __init__(self, flist=None, filename=None, key=None, root=None):
         # Delay import: runscript imports pyshell imports EditorWindow.
@@ -784,9 +784,7 @@ def _addcolorizer(self):
             self.color = self.ColorDelegator()
         # can add more colorizers here...
         if self.color:
-            self.per.removefilter(self.undo)
-            self.per.insertfilter(self.color)
-            self.per.insertfilter(self.undo)
+            self.per.insertfilterafter(filter=self.color, after=self.undo)
 
     def _rmcolorizer(self):
         if not self.color:
@@ -1303,8 +1301,6 @@ def smart_backspace_event(self, event):
         # Debug prompt is multilined....
         ncharsdeleted = 0
         while 1:
-            if chars == self.prompt_last_line:  # '' unless PyShell
-                break
             chars = chars[:-1]
             ncharsdeleted = ncharsdeleted + 1
             have = len(chars.expandtabs(tabwidth))
@@ -1313,7 +1309,8 @@ def smart_backspace_event(self, event):
         text.undo_block_start()
         text.delete("insert-%dc" % ncharsdeleted, "insert")
         if have < want:
-            text.insert("insert", ' ' * (want - have))
+            text.insert("insert", ' ' * (want - have),
+                        self.user_input_insert_tags)
         text.undo_block_stop()
         return "break"
 
@@ -1346,7 +1343,7 @@ def smart_indent_event(self, event):
                     effective = len(prefix.expandtabs(self.tabwidth))
                     n = self.indentwidth
                     pad = ' ' * (n - effective % n)
-                text.insert("insert", pad)
+                text.insert("insert", pad, self.user_input_insert_tags)
             text.see("insert")
             return "break"
         finally:
@@ -1377,13 +1374,14 @@ def newline_and_indent_event(self, event):
             if i == n:
                 # The cursor is in or at leading indentation in a continuation
                 # line; just inject an empty line at the start.
-                text.insert("insert linestart", '\n')
+                text.insert("insert linestart", '\n',
+                            self.user_input_insert_tags)
                 return "break"
             indent = line[:i]
 
             # Strip whitespace before insert point unless it's in the prompt.
             i = 0
-            while line and line[-1] in " \t" and line != self.prompt_last_line:
+            while line and line[-1] in " \t":
                 line = line[:-1]
                 i += 1
             if i:
@@ -1394,7 +1392,7 @@ def newline_and_indent_event(self, event):
                 text.delete("insert")
 
             # Insert new line.
-            text.insert("insert", '\n')
+            text.insert("insert", '\n', self.user_input_insert_tags)
 
             # Adjust indentation for continuations and block open/close.
             # First need to find the last statement.
@@ -1430,7 +1428,7 @@ def newline_and_indent_event(self, event):
                 elif c == pyparse.C_STRING_NEXT_LINES:
                     # Inside a string which started before this line;
                     # just mimic the current indent.
-                    text.insert("insert", indent)
+                    text.insert("insert", indent, self.user_input_insert_tags)
                 elif c == pyparse.C_BRACKET:
                     # Line up with the first (if any) element of the
                     # last open bracket structure; else indent one
@@ -1444,7 +1442,8 @@ def newline_and_indent_event(self, event):
                     # beyond leftmost =; else to beyond first chunk of
                     # non-whitespace on initial line.
                     if y.get_num_lines_in_stmt() > 1:
-                        text.insert("insert", indent)
+                        text.insert("insert", indent,
+                                    self.user_input_insert_tags)
                     else:
                         self.reindent_to(y.compute_backslash_indent())
                 else:
@@ -1455,7 +1454,7 @@ def newline_and_indent_event(self, event):
             # indentation of initial line of closest preceding
             # interesting statement.
             indent = y.get_base_indent_string()
-            text.insert("insert", indent)
+            text.insert("insert", indent, self.user_input_insert_tags)
             if y.is_block_opener():
                 self.smart_indent_event(event)
             elif indent and y.is_block_closer():
@@ -1502,7 +1501,8 @@ def reindent_to(self, column):
         if text.compare("insert linestart", "!=", "insert"):
             text.delete("insert linestart", "insert")
         if column:
-            text.insert("insert", self._make_blanks(column))
+            text.insert("insert", self._make_blanks(column),
+                        self.user_input_insert_tags)
         text.undo_block_stop()
 
     # Guess indentwidth from text content.

Lib/idlelib/history.py

@@ -74,13 +74,13 @@ def fetch(self, reverse):
                 else:
                     if self.text.get("iomark", "end-1c") != prefix:
                         self.text.delete("iomark", "end-1c")
-                        self.text.insert("iomark", prefix)
+                        self.text.insert("iomark", prefix, "stdin")
                     pointer = prefix = None
                 break
             item = self.history[pointer]
             if item[:nprefix] == prefix and len(item) > nprefix:
                 self.text.delete("iomark", "end-1c")
-                self.text.insert("iomark", item)
+                self.text.insert("iomark", item, "stdin")
                 break
         self.text.see("insert")
         self.text.tag_remove("sel", "1.0", "end")

Lib/idlelib/idle_test/test_editor.py

@@ -167,7 +167,6 @@ def test_indent_and_newline_event(self):
                           '2.end'),
                  )
 
-        w.prompt_last_line = ''
         for test in tests:
             with self.subTest(label=test.label):
                 insert(text, test.text)
@@ -182,13 +181,6 @@ def test_indent_and_newline_event(self):
         # Deletes selected text before adding new line.
         eq(get('1.0', 'end'), '  def f1(self, a,\n         \n    return a + b\n')
 
-        # Preserves the whitespace in shell prompt.
-        w.prompt_last_line = '>>> '
-        insert(text, '>>> \t\ta =')
-        text.mark_set('insert', '1.5')
-        nl(None)
-        eq(get('1.0', 'end'), '>>> \na =\n')
-
 
 class RMenuTest(unittest.TestCase):
 

Lib/idlelib/idle_test/test_pyshell.py

@@ -60,5 +60,89 @@ def test_init(self):
 ##        self.assertIsInstance(ps, pyshell.PyShell)
 
 
+class PyShellRemoveLastNewlineAndSurroundingWhitespaceTest(unittest.TestCase):
+    regexp = pyshell.PyShell._last_newline_re
+
+    def all_removed(self, text):
+        self.assertEqual('', self.regexp.sub('', text))
+
+    def none_removed(self, text):
+        self.assertEqual(text, self.regexp.sub('', text))
+
+    def check_result(self, text, expected):
+        self.assertEqual(expected, self.regexp.sub('', text))
+
+    def test_empty(self):
+        self.all_removed('')
+
+    def test_newline(self):
+        self.all_removed('\n')
+
+    def test_whitespace_no_newline(self):
+        self.all_removed(' ')
+        self.all_removed('  ')
+        self.all_removed('   ')
+        self.all_removed(' ' * 20)
+        self.all_removed('\t')
+        self.all_removed('\t\t')
+        self.all_removed('\t\t\t')
+        self.all_removed('\t' * 20)
+        self.all_removed('\t ')
+        self.all_removed(' \t')
+        self.all_removed(' \t \t ')
+        self.all_removed('\t \t \t')
+
+    def test_newline_with_whitespace(self):
+        self.all_removed(' \n')
+        self.all_removed('\t\n')
+        self.all_removed(' \t\n')
+        self.all_removed('\t \n')
+        self.all_removed('\n ')
+        self.all_removed('\n\t')
+        self.all_removed('\n \t')
+        self.all_removed('\n\t ')
+        self.all_removed(' \n ')
+        self.all_removed('\t\n ')
+        self.all_removed(' \n\t')
+        self.all_removed('\t\n\t')
+        self.all_removed('\t \t \t\n')
+        self.all_removed(' \t \t \n')
+        self.all_removed('\n\t \t \t')
+        self.all_removed('\n \t \t ')
+
+    def test_multiple_newlines(self):
+        self.check_result('\n\n', '\n')
+        self.check_result('\n' * 5, '\n' * 4)
+        self.check_result('\n' * 5 + '\t', '\n' * 4)
+        self.check_result('\n' * 20, '\n' * 19)
+        self.check_result('\n' * 20 + ' ', '\n' * 19)
+        self.check_result(' \n \n ', ' \n')
+        self.check_result(' \n\n ', ' \n')
+        self.check_result(' \n\n', ' \n')
+        self.check_result('\t\n\n', '\t\n')
+        self.check_result('\n\n ', '\n')
+        self.check_result('\n\n\t', '\n')
+        self.check_result(' \n \n ', ' \n')
+        self.check_result('\t\n\t\n\t', '\t\n')
+
+    def test_non_whitespace(self):
+        self.none_removed('a')
+        self.check_result('a\n', 'a')
+        self.check_result('a\n ', 'a')
+        self.check_result('a \n ', 'a')
+        self.check_result('a \n\t', 'a')
+        self.none_removed('-')
+        self.check_result('-\n', '-')
+        self.none_removed('.')
+        self.check_result('.\n', '.')
+
+    def test_unsupported_whitespace(self):
+        self.none_removed('\v')
+        self.none_removed('\n\v')
+        self.check_result('\v\n', '\v')
+        self.none_removed(' \n\v')
+        self.check_result('\v\n ', '\v')
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)

Lib/idlelib/idle_test/test_sidebar.py

@@ -7,13 +7,12 @@
 from test.support import requires
 
 from idlelib.config import idleConf
+from idlelib.percolator import Percolator
 from idlelib.squeezer import count_lines_with_wrapping, ExpandingButton, \
     Squeezer
 from idlelib import macosx
 from idlelib.textview import view_text
 from idlelib.tooltip import Hovertip
-from idlelib.pyshell import PyShell
-
 
 SENTINEL_VALUE = sentinel.SENTINEL_VALUE
 
@@ -205,8 +204,8 @@ def test_auto_squeeze(self):
         self.assertEqual(text_widget.get('1.0', 'end'), '\n')
         self.assertEqual(len(squeezer.expandingbuttons), 1)
 
-    def test_squeeze_current_text_event(self):
-        """Test the squeeze_current_text event."""
+    def test_squeeze_current_text(self):
+        """Test the squeeze_current_text method."""
         # Squeezing text should work for both stdout and stderr.
         for tag_name in ["stdout", "stderr"]:
             editwin = self.make_mock_editor_window(with_text_widget=True)
@@ -222,19 +221,19 @@ def test_squeeze_current_text_event(self):
             self.assertEqual(len(squeezer.expandingbuttons), 0)
 
             # Test squeezing the current text.
-            retval = squeezer.squeeze_current_text_event(event=Mock())
+            retval = squeezer.squeeze_current_text()
             self.assertEqual(retval, "break")
             self.assertEqual(text_widget.get('1.0', 'end'), '\n\n')
             self.assertEqual(len(squeezer.expandingbuttons), 1)
             self.assertEqual(squeezer.expandingbuttons[0].s, 'SOME\nTEXT')
 
             # Test that expanding the squeezed text works and afterwards
             # the Text widget contains the original text.
-            squeezer.expandingbuttons[0].expand(event=Mock())
+            squeezer.expandingbuttons[0].expand()
             self.assertEqual(text_widget.get('1.0', 'end'), 'SOME\nTEXT\n\n')
             self.assertEqual(len(squeezer.expandingbuttons), 0)
 
-    def test_squeeze_current_text_event_no_allowed_tags(self):
+    def test_squeeze_current_text_no_allowed_tags(self):
         """Test that the event doesn't squeeze text without a relevant tag."""
         editwin = self.make_mock_editor_window(with_text_widget=True)
         text_widget = editwin.text
@@ -249,7 +248,7 @@ def test_squeeze_current_text_event_no_allowed_tags(self):
         self.assertEqual(len(squeezer.expandingbuttons), 0)
 
         # Test squeezing the current text.
-        retval = squeezer.squeeze_current_text_event(event=Mock())
+        retval = squeezer.squeeze_current_text()
         self.assertEqual(retval, "break")
         self.assertEqual(text_widget.get('1.0', 'end'), 'SOME\nTEXT\n\n')
         self.assertEqual(len(squeezer.expandingbuttons), 0)
@@ -264,13 +263,13 @@ def test_squeeze_text_before_existing_squeezed_text(self):
         # Prepare some text in the Text widget and squeeze it.
         text_widget.insert("1.0", "SOME\nTEXT\n", "stdout")
         text_widget.mark_set("insert", "1.0")
-        squeezer.squeeze_current_text_event(event=Mock())
+        squeezer.squeeze_current_text()
         self.assertEqual(len(squeezer.expandingbuttons), 1)
 
         # Test squeezing the current text.
         text_widget.insert("1.0", "MORE\nSTUFF\n", "stdout")
         text_widget.mark_set("insert", "1.0")
-        retval = squeezer.squeeze_current_text_event(event=Mock())
+        retval = squeezer.squeeze_current_text()
         self.assertEqual(retval, "break")
         self.assertEqual(text_widget.get('1.0', 'end'), '\n\n\n')
         self.assertEqual(len(squeezer.expandingbuttons), 2)
@@ -311,6 +310,7 @@ def make_mock_squeezer(self):
         root = get_test_tk_root(self)
         squeezer = Mock()
         squeezer.editwin.text = Text(root)
+        squeezer.editwin.per = Percolator(squeezer.editwin.text)
 
         # Set default values for the configuration settings.
         squeezer.auto_squeeze_min_lines = 50
@@ -352,14 +352,9 @@ def test_expand(self):
 
         # Insert the button into the text widget
         # (this is normally done by the Squeezer class).
-        text_widget = expandingbutton.text
+        text_widget = squeezer.editwin.text
         text_widget.window_create("1.0", window=expandingbutton)
 
-        # Set base_text to the text widget, so that changes are actually
-        # made to it (by ExpandingButton) and we can inspect these
-        # changes afterwards.
-        expandingbutton.base_text = expandingbutton.text
-
         # trigger the expand event
         retval = expandingbutton.expand(event=Mock())
         self.assertEqual(retval, None)
@@ -390,11 +385,6 @@ def test_expand_dangerous_oupput(self):
         text_widget = expandingbutton.text
         text_widget.window_create("1.0", window=expandingbutton)
 
-        # Set base_text to the text widget, so that changes are actually
-        # made to it (by ExpandingButton) and we can inspect these
-        # changes afterwards.
-        expandingbutton.base_text = expandingbutton.text
-
         # Patch the message box module to always return False.
         with patch('idlelib.squeezer.messagebox') as mock_msgbox:
             mock_msgbox.askokcancel.return_value = False

Lib/idlelib/idle_test/test_squeezer.py

@@ -0,0 +1,56 @@
+"""Utilities for testing with Tkinter"""
+import functools
+
+
+def run_in_tk_mainloop(test_method):
+    """Decorator for running a test method with a real Tk mainloop.
+
+    This starts a Tk mainloop before running the test, and stops it
+    at the end. This is faster and more robust than the common
+    alternative method of calling .update() and/or .update_idletasks().
+
+    Test methods using this must be written as generator functions,
+    using "yield" to allow the mainloop to process events and "after"
+    callbacks, and then continue the test from that point.
+
+    This also assumes that the test class has a .root attribute,
+    which is a tkinter.Tk object.
+
+    For example (from test_sidebar.py):
+
+    @run_test_with_tk_mainloop
+    def test_single_empty_input(self):
+        self.do_input('\n')
+        yield
+        self.assert_sidebar_lines_end_with(['>>>', '>>>'])
+    """
+    @functools.wraps(test_method)
+    def new_test_method(self):
+        test_generator = test_method(self)
+        root = self.root
+        # Exceptions raised by self.assert...() need to be raised
+        # outside of the after() callback in order for the test
+        # harness to capture them.
+        exception = None
+        def after_callback():
+            nonlocal exception
+            try:
+                next(test_generator)
+            except StopIteration:
+                root.quit()
+            except Exception as exc:
+                exception = exc
+                root.quit()
+            else:
+                # Schedule the Tk mainloop to call this function again,
+                # using a robust method of ensuring that it gets a
+                # chance to process queued events before doing so.
+                # See: https://stackoverflow.com/q/18499082#comment65004099_38817470
+                root.after(1, root.after_idle, after_callback)
+        root.after(0, root.after_idle, after_callback)
+        root.mainloop()
+
+        if exception:
+            raise exception
+
+    return new_test_method

Lib/idlelib/idle_test/tkinter_testing_utils.py

@@ -38,6 +38,21 @@ def insertfilter(self, filter):
         filter.setdelegate(self.top)
         self.top = filter
 
+    def insertfilterafter(self, filter, after):
+        assert isinstance(filter, Delegator)
+        assert isinstance(after, Delegator)
+        assert filter.delegate is None
+
+        f = self.top
+        f.resetcache()
+        while f is not after:
+            assert f is not self.bottom
+            f = f.delegate
+            f.resetcache()
+
+        filter.setdelegate(f.delegate)
+        f.setdelegate(filter)
+
     def removefilter(self, filter):
         # XXX Perhaps should only support popfilter()?
         assert isinstance(filter, Delegator)

Lib/idlelib/percolator.py

@@ -48,15 +48,20 @@
 
 from idlelib.colorizer import ColorDelegator
 from idlelib.config import idleConf
+from idlelib.delegator import Delegator
 from idlelib import debugger
 from idlelib import debugger_r
 from idlelib.editor import EditorWindow, fixwordbreaks
 from idlelib.filelist import FileList
 from idlelib.outwin import OutputWindow
+from idlelib import replace
 from idlelib import rpc
 from idlelib.run import idle_formatwarning, StdInputFile, StdOutputFile
 from idlelib.undo import UndoDelegator
 
+# Default for testing; defaults to True in main() for running.
+use_subprocess = False
+
 HOST = '127.0.0.1' # python execution server on localhost loopback
 PORT = 0  # someday pass in host, port for remote debug capability
 
@@ -335,34 +340,19 @@ def open_shell(self, event=None):
 
 class ModifiedColorDelegator(ColorDelegator):
     "Extend base class: colorizer for the shell window itself"
-
-    def __init__(self):
-        ColorDelegator.__init__(self)
-        self.LoadTagDefs()
-
     def recolorize_main(self):
         self.tag_remove("TODO", "1.0", "iomark")
         self.tag_add("SYNC", "1.0", "iomark")
         ColorDelegator.recolorize_main(self)
 
-    def LoadTagDefs(self):
-        ColorDelegator.LoadTagDefs(self)
-        theme = idleConf.CurrentTheme()
-        self.tagdefs.update({
-            "stdin": {'background':None,'foreground':None},
-            "stdout": idleConf.GetHighlight(theme, "stdout"),
-            "stderr": idleConf.GetHighlight(theme, "stderr"),
-            "console": idleConf.GetHighlight(theme, "console"),
-        })
-
     def removecolors(self):
         # Don't remove shell color tags before "iomark"
         for tag in self.tagdefs:
             self.tag_remove(tag, "iomark", "end")
 
+
 class ModifiedUndoDelegator(UndoDelegator):
     "Extend base class: forbid insert/delete before the I/O mark"
-
     def insert(self, index, chars, tags=None):
         try:
             if self.delegate.compare(index, "<", "iomark"):
@@ -381,6 +371,27 @@ def delete(self, index1, index2=None):
             pass
         UndoDelegator.delete(self, index1, index2)
 
+    def undo_event(self, event):
+        # Temporarily monkey-patch the delegate's .insert() method to
+        # always use the "stdin" tag.  This is needed for undo-ing
+        # deletions to preserve the "stdin" tag, because UndoDelegator
+        # doesn't preserve tags for deleted text.
+        orig_insert = self.delegate.insert
+        self.delegate.insert = \
+            lambda index, chars: orig_insert(index, chars, "stdin")
+        try:
+            super().undo_event(event)
+        finally:
+            self.delegate.insert = orig_insert
+
+
+class UserInputTaggingDelegator(Delegator):
+    """Delegator used to tag user input with "stdin"."""
+    def insert(self, index, chars, tags=None):
+        if tags is None:
+            tags = "stdin"
+        self.delegate.insert(index, chars, tags)
+
 
 class MyRPCClient(rpc.RPCClient):
 
@@ -832,6 +843,7 @@ def display_executing_dialog(self):
 
 
 class PyShell(OutputWindow):
+    from idlelib.squeezer import Squeezer
 
     shell_title = "IDLE Shell " + python_version()
 
@@ -855,9 +867,11 @@ class PyShell(OutputWindow):
     ]
 
     allow_line_numbers = False
+    user_input_insert_tags = "stdin"
 
     # New classes
     from idlelib.history import History
+    from idlelib.sidebar import ShellSidebar
 
     def __init__(self, flist=None):
         if use_subprocess:
@@ -871,6 +885,8 @@ def __init__(self, flist=None):
             root.withdraw()
             flist = PyShellFileList(root)
 
+        self.shell_sidebar = None  # initialized below
+
         OutputWindow.__init__(self, flist, None, None)
 
         self.usetabs = True
@@ -893,9 +909,9 @@ def __init__(self, flist=None):
         if use_subprocess:
             text.bind("<<view-restart>>", self.view_restart_mark)
             text.bind("<<restart-shell>>", self.restart_shell)
-        squeezer = self.Squeezer(self)
+        self.squeezer = self.Squeezer(self)
         text.bind("<<squeeze-current-text>>",
-                  squeezer.squeeze_current_text_event)
+                  self.squeeze_current_text_event)
 
         self.save_stdout = sys.stdout
         self.save_stderr = sys.stderr
@@ -926,6 +942,40 @@ def __init__(self, flist=None):
         #
         self.pollinterval = 50  # millisec
 
+        self.shell_sidebar = self.ShellSidebar(self)
+
+        # Insert UserInputTaggingDelegator at the top of the percolator,
+        # but make calls to text.insert() skip it.  This causes only insert
+        # events generated in Tcl/Tk to go through this delegator.
+        self.text.insert = self.per.top.insert
+        self.per.insertfilter(UserInputTaggingDelegator())
+
+    def ResetFont(self):
+        super().ResetFont()
+
+        if self.shell_sidebar is not None:
+            self.shell_sidebar.update_font()
+
+    def ResetColorizer(self):
+        super().ResetColorizer()
+
+        theme = idleConf.CurrentTheme()
+        tag_colors = {
+          "stdin": {'background': None, 'foreground': None},
+          "stdout": idleConf.GetHighlight(theme, "stdout"),
+          "stderr": idleConf.GetHighlight(theme, "stderr"),
+          "console": idleConf.GetHighlight(theme, "normal"),
+        }
+        for tag, tag_colors_config in tag_colors.items():
+            self.text.tag_configure(tag, **tag_colors_config)
+
+        if self.shell_sidebar is not None:
+            self.shell_sidebar.update_colors()
+
+    def replace_event(self, event):
+        replace.replace(self.text, insert_tags="stdin")
+        return "break"
+
     def get_standard_extension_names(self):
         return idleConf.GetExtensions(shell_only=True)
 
@@ -1166,13 +1216,30 @@ def enter_callback(self, event):
         # the current line, less a leading prompt, less leading or
         # trailing whitespace
         if self.text.compare("insert", "<", "iomark linestart"):
-            # Check if there's a relevant stdin range -- if so, use it
+            # Check if there's a relevant stdin range -- if so, use it.
+            # Note: "stdin" blocks may include several successive statements,
+            # so look for "console" tags on the newline before each statement
+            # (and possibly on prompts).
             prev = self.text.tag_prevrange("stdin", "insert")
-            if prev and self.text.compare("insert", "<", prev[1]):
+            if (
+                    prev and
+                    self.text.compare("insert", "<", prev[1]) and
+                    # The following is needed to handle empty statements.
+                    "console" not in self.text.tag_names("insert")
+            ):
+                prev_cons = self.text.tag_prevrange("console", "insert")
+                if prev_cons and self.text.compare(prev_cons[1], ">=", prev[0]):
+                    prev = (prev_cons[1], prev[1])
+                next_cons = self.text.tag_nextrange("console", "insert")
+                if next_cons and self.text.compare(next_cons[0], "<", prev[1]):
+                    prev = (prev[0], self.text.index(next_cons[0] + "+1c"))
                 self.recall(self.text.get(prev[0], prev[1]), event)
                 return "break"
             next = self.text.tag_nextrange("stdin", "insert")
             if next and self.text.compare("insert lineend", ">=", next[0]):
+                next_cons = self.text.tag_nextrange("console", "insert lineend")
+                if next_cons and self.text.compare(next_cons[0], "<", next[1]):
+                    next = (next[0], self.text.index(next_cons[0] + "+1c"))
                 self.recall(self.text.get(next[0], next[1]), event)
                 return "break"
             # No stdin mark -- just get the current line, less any prompt
@@ -1204,7 +1271,6 @@ def enter_callback(self, event):
             self.text.see("insert")
         else:
             self.newline_and_indent_event(event)
-        self.text.tag_add("stdin", "iomark", "end-1c")
         self.text.update_idletasks()
         if self.reading:
             self.top.quit() # Break out of recursive mainloop()
@@ -1214,7 +1280,7 @@ def enter_callback(self, event):
 
     def recall(self, s, event):
         # remove leading and trailing empty or whitespace lines
-        s = re.sub(r'^\s*\n', '' , s)
+        s = re.sub(r'^\s*\n', '', s)
         s = re.sub(r'\n\s*$', '', s)
         lines = s.split('\n')
         self.text.undo_block_start()
@@ -1225,32 +1291,33 @@ def recall(self, s, event):
             if prefix.rstrip().endswith(':'):
                 self.newline_and_indent_event(event)
                 prefix = self.text.get("insert linestart", "insert")
-            self.text.insert("insert", lines[0].strip())
+            self.text.insert("insert", lines[0].strip(),
+                             self.user_input_insert_tags)
             if len(lines) > 1:
                 orig_base_indent = re.search(r'^([ \t]*)', lines[0]).group(0)
                 new_base_indent  = re.search(r'^([ \t]*)', prefix).group(0)
                 for line in lines[1:]:
                     if line.startswith(orig_base_indent):
                         # replace orig base indentation with new indentation
                         line = new_base_indent + line[len(orig_base_indent):]
-                    self.text.insert('insert', '\n'+line.rstrip())
+                    self.text.insert('insert', '\n' + line.rstrip(),
+                                     self.user_input_insert_tags)
         finally:
             self.text.see("insert")
             self.text.undo_block_stop()
 
+    _last_newline_re = re.compile(r"[ \t]*(\n[ \t]*)?\Z")
     def runit(self):
+        index_before = self.text.index("end-2c")
         line = self.text.get("iomark", "end-1c")
         # Strip off last newline and surrounding whitespace.
         # (To allow you to hit return twice to end a statement.)
-        i = len(line)
-        while i > 0 and line[i-1] in " \t":
-            i = i-1
-        if i > 0 and line[i-1] == "\n":
-            i = i-1
-        while i > 0 and line[i-1] in " \t":
-            i = i-1
-        line = line[:i]
-        self.interp.runsource(line)
+        line = self._last_newline_re.sub("", line)
+        input_is_complete = self.interp.runsource(line)
+        if not input_is_complete:
+            if self.text.get(index_before) == '\n':
+                self.text.tag_remove(self.user_input_insert_tags, index_before)
+            self.shell_sidebar.update_sidebar()
 
     def open_stack_viewer(self, event=None):
         if self.interp.rpcclt:
@@ -1276,7 +1343,14 @@ def restart_shell(self, event=None):
 
     def showprompt(self):
         self.resetoutput()
-        self.console.write(self.prompt)
+
+        prompt = self.prompt
+        if self.sys_ps1 and prompt.endswith(self.sys_ps1):
+            prompt = prompt[:-len(self.sys_ps1)]
+        self.text.tag_add("console", "iomark-1c")
+        self.console.write(prompt)
+
+        self.shell_sidebar.update_sidebar()
         self.text.mark_set("insert", "end-1c")
         self.set_line_and_column()
         self.io.reset_undo()
@@ -1326,6 +1400,13 @@ def rmenu_check_paste(self):
             return 'disabled'
         return super().rmenu_check_paste()
 
+    def squeeze_current_text_event(self, event=None):
+        self.squeezer.squeeze_current_text()
+        self.shell_sidebar.update_sidebar()
+
+    def on_squeezed_expand(self, index, text, tags):
+        self.shell_sidebar.update_sidebar()
+
 
 def fix_x11_paste(root):
     "Make paste replace selection on x11.  See issue #5124."

Lib/idlelib/pyshell.py

@@ -11,7 +11,7 @@
 from idlelib import searchengine
 
 
-def replace(text):
+def replace(text, insert_tags=None):
     """Create or reuse a singleton ReplaceDialog instance.
 
     The singleton dialog saves user entries and preferences
@@ -25,7 +25,7 @@ def replace(text):
     if not hasattr(engine, "_replacedialog"):
         engine._replacedialog = ReplaceDialog(root, engine)
     dialog = engine._replacedialog
-    dialog.open(text)
+    dialog.open(text, insert_tags=insert_tags)
 
 
 class ReplaceDialog(SearchDialogBase):
@@ -49,8 +49,9 @@ def __init__(self, root, engine):
         """
         super().__init__(root, engine)
         self.replvar = StringVar(root)
+        self.insert_tags = None
 
-    def open(self, text):
+    def open(self, text, insert_tags=None):
         """Make dialog visible on top of others and ready to use.
 
         Also, highlight the currently selected text and set the
@@ -72,6 +73,7 @@ def open(self, text):
         last = last or first
         self.show_hit(first, last)
         self.ok = True
+        self.insert_tags = insert_tags
 
     def create_entries(self):
         "Create base and additional label and text entry widgets."
@@ -177,7 +179,7 @@ def replace_all(self, event=None):
                 if first != last:
                     text.delete(first, last)
                 if new:
-                    text.insert(first, new)
+                    text.insert(first, new, self.insert_tags)
             col = i + len(new)
             ok = False
         text.undo_block_stop()
@@ -231,7 +233,7 @@ def do_replace(self):
         if m.group():
             text.delete(first, last)
         if new:
-            text.insert(first, new)
+            text.insert(first, new, self.insert_tags)
         text.undo_block_stop()
         self.show_hit(first, text.index("insert"))
         self.ok = False
@@ -264,6 +266,7 @@ def close(self, event=None):
         "Close the dialog and remove hit tags."
         SearchDialogBase.close(self, event)
         self.text.tag_remove("hit", "1.0", "end")
+        self.insert_tags = None
 
 
 def _replace_dialog(parent):  # htest #

Lib/idlelib/replace.py

@@ -1,19 +1,33 @@
 """Line numbering implementation for IDLE as an extension.
 Includes BaseSideBar which can be extended for other sidebar based extensions
 """
+import contextlib
 import functools
 import itertools
 
 import tkinter as tk
+from tkinter.font import Font
 from idlelib.config import idleConf
 from idlelib.delegator import Delegator
 
 
+def get_lineno(text, index):
+    """Return the line number of an index in a Tk text widget."""
+    return int(float(text.index(index)))
+
+
 def get_end_linenumber(text):
-    """Utility to get the last line's number in a Tk text widget."""
-    return int(float(text.index('end-1c')))
+    """Return the number of the last line in a Tk text widget."""
+    return get_lineno(text, 'end-1c')
 
 
+def get_displaylines(text, index):
+    """Display height, in lines, of a logical line in a Tk text widget."""
+    res = text.count(f"{index} linestart",
+                     f"{index} lineend",
+                     "displaylines")
+    return res[0] if res else 0
+
 def get_widget_padding(widget):
     """Get the total padding of a Tk widget, including its border."""
     # TODO: use also in codecontext.py
@@ -40,10 +54,17 @@ def get_widget_padding(widget):
     return padx, pady
 
 
+@contextlib.contextmanager
+def temp_enable_text_widget(text):
+    text.configure(state=tk.NORMAL)
+    try:
+        yield
+    finally:
+        text.configure(state=tk.DISABLED)
+
+
 class BaseSideBar:
-    """
-    The base class for extensions which require a sidebar.
-    """
+    """A base class for sidebars using Text."""
     def __init__(self, editwin):
         self.editwin = editwin
         self.parent = editwin.text_frame
@@ -119,14 +140,11 @@ def redirect_mousewheel_event(self, event):
 
 
 class EndLineDelegator(Delegator):
-    """Generate callbacks with the current end line number after
-       insert or delete operations"""
+    """Generate callbacks with the current end line number.
+
+    The provided callback is called after every insert and delete.
+    """
     def __init__(self, changed_callback):
-        """
-        changed_callback - Callable, will be called after insert
-                           or delete operations with the current
-                           end line number.
-        """
         Delegator.__init__(self)
         self.changed_callback = changed_callback
 
@@ -159,16 +177,8 @@ def __init__(self, editwin):
         end_line_delegator = EndLineDelegator(self.update_sidebar_text)
         # Insert the delegator after the undo delegator, so that line numbers
         # are properly updated after undo and redo actions.
-        end_line_delegator.setdelegate(self.editwin.undo.delegate)
-        self.editwin.undo.setdelegate(end_line_delegator)
-        # Reset the delegator caches of the delegators "above" the
-        # end line delegator we just inserted.
-        delegator = self.editwin.per.top
-        while delegator is not end_line_delegator:
-            delegator.resetcache()
-            delegator = delegator.delegate
-
-        self.is_shown = False
+        self.editwin.per.insertfilterafter(filter=end_line_delegator,
+                                           after=self.editwin.undo)
 
     def bind_events(self):
         # Ensure focus is always redirected to the main editor text widget.
@@ -297,20 +307,209 @@ def update_sidebar_text(self, end):
             new_width = cur_width + width_difference
             self.sidebar_text['width'] = self._sidebar_width_type(new_width)
 
-        self.sidebar_text.config(state=tk.NORMAL)
-        if end > self.prev_end:
-            new_text = '\n'.join(itertools.chain(
-                [''],
-                map(str, range(self.prev_end + 1, end + 1)),
-            ))
-            self.sidebar_text.insert(f'end -1c', new_text, 'linenumber')
-        else:
-            self.sidebar_text.delete(f'{end+1}.0 -1c', 'end -1c')
-        self.sidebar_text.config(state=tk.DISABLED)
+        with temp_enable_text_widget(self.sidebar_text):
+            if end > self.prev_end:
+                new_text = '\n'.join(itertools.chain(
+                    [''],
+                    map(str, range(self.prev_end + 1, end + 1)),
+                ))
+                self.sidebar_text.insert(f'end -1c', new_text, 'linenumber')
+            else:
+                self.sidebar_text.delete(f'{end+1}.0 -1c', 'end -1c')
 
         self.prev_end = end
 
 
+class WrappedLineHeightChangeDelegator(Delegator):
+    def __init__(self, callback):
+        """
+        callback - Callable, will be called when an insert, delete or replace
+                   action on the text widget may require updating the shell
+                   sidebar.
+        """
+        Delegator.__init__(self)
+        self.callback = callback
+
+    def insert(self, index, chars, tags=None):
+        is_single_line = '\n' not in chars
+        if is_single_line:
+            before_displaylines = get_displaylines(self, index)
+
+        self.delegate.insert(index, chars, tags)
+
+        if is_single_line:
+            after_displaylines = get_displaylines(self, index)
+            if after_displaylines == before_displaylines:
+                return  # no need to update the sidebar
+
+        self.callback()
+
+    def delete(self, index1, index2=None):
+        if index2 is None:
+            index2 = index1 + "+1c"
+        is_single_line = get_lineno(self, index1) == get_lineno(self, index2)
+        if is_single_line:
+            before_displaylines = get_displaylines(self, index1)
+
+        self.delegate.delete(index1, index2)
+
+        if is_single_line:
+            after_displaylines = get_displaylines(self, index1)
+            if after_displaylines == before_displaylines:
+                return  # no need to update the sidebar
+
+        self.callback()
+
+
+class ShellSidebar:
+    """Sidebar for the PyShell window, for prompts etc."""
+    def __init__(self, editwin):
+        self.editwin = editwin
+        self.parent = editwin.text_frame
+        self.text = editwin.text
+
+        self.canvas = tk.Canvas(self.parent, width=30,
+                                borderwidth=0, highlightthickness=0,
+                                takefocus=False)
+
+        self.bind_events()
+
+        change_delegator = \
+            WrappedLineHeightChangeDelegator(self.change_callback)
+
+        # Insert the TextChangeDelegator after the last delegator, so that
+        # the sidebar reflects final changes to the text widget contents.
+        d = self.editwin.per.top
+        if d.delegate is not self.text:
+            while d.delegate is not self.editwin.per.bottom:
+                d = d.delegate
+        self.editwin.per.insertfilterafter(change_delegator, after=d)
+
+        self.text['yscrollcommand'] = self.yscroll_event
+
+        self.is_shown = False
+
+        self.update_font()
+        self.update_colors()
+        self.update_sidebar()
+        self.canvas.grid(row=1, column=0, sticky=tk.NSEW, padx=2, pady=0)
+        self.is_shown = True
+
+    def change_callback(self):
+        if self.is_shown:
+            self.update_sidebar()
+
+    def update_sidebar(self):
+        text = self.text
+        text_tagnames = text.tag_names
+        canvas = self.canvas
+
+        canvas.delete(tk.ALL)
+
+        index = text.index("@0,0")
+        if index.split('.', 1)[1] != '0':
+            index = text.index(f'{index}+1line linestart')
+        while True:
+            lineinfo = text.dlineinfo(index)
+            if lineinfo is None:
+                break
+            y = lineinfo[1]
+            prev_newline_tagnames = text_tagnames(f"{index} linestart -1c")
+            prompt = (
+                '>>>' if "console" in prev_newline_tagnames else
+                '...' if "stdin" in prev_newline_tagnames else
+                None
+            )
+            if prompt:
+                canvas.create_text(2, y, anchor=tk.NW, text=prompt,
+                                   font=self.font, fill=self.colors[0])
+            index = text.index(f'{index}+1line')
+
+    def yscroll_event(self, *args, **kwargs):
+        """Redirect vertical scrolling to the main editor text widget.
+
+        The scroll bar is also updated.
+        """
+        self.editwin.vbar.set(*args)
+        self.change_callback()
+        return 'break'
+
+    def update_font(self):
+        """Update the sidebar text font, usually after config changes."""
+        font = idleConf.GetFont(self.text, 'main', 'EditorWindow')
+        tk_font = Font(self.text, font=font)
+        char_width = max(tk_font.measure(char) for char in ['>', '.'])
+        self.canvas.configure(width=char_width * 3 + 4)
+        self._update_font(font)
+
+    def _update_font(self, font):
+        self.font = font
+        self.change_callback()
+
+    def update_colors(self):
+        """Update the sidebar text colors, usually after config changes."""
+        linenumbers_colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'linenumber')
+        prompt_colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'console')
+        self._update_colors(foreground=prompt_colors['foreground'],
+                            background=linenumbers_colors['background'])
+
+    def _update_colors(self, foreground, background):
+        self.colors = (foreground, background)
+        self.canvas.configure(background=self.colors[1])
+        self.change_callback()
+
+    def redirect_focusin_event(self, event):
+        """Redirect focus-in events to the main editor text widget."""
+        self.text.focus_set()
+        return 'break'
+
+    def redirect_mousebutton_event(self, event, event_name):
+        """Redirect mouse button events to the main editor text widget."""
+        self.text.focus_set()
+        self.text.event_generate(event_name, x=0, y=event.y)
+        return 'break'
+
+    def redirect_mousewheel_event(self, event):
+        """Redirect mouse wheel events to the editwin text widget."""
+        self.text.event_generate('<MouseWheel>',
+                                 x=0, y=event.y, delta=event.delta)
+        return 'break'
+
+    def bind_events(self):
+        # Ensure focus is always redirected to the main editor text widget.
+        self.canvas.bind('<FocusIn>', self.redirect_focusin_event)
+
+        # Redirect mouse scrolling to the main editor text widget.
+        #
+        # Note that without this, scrolling with the mouse only scrolls
+        # the line numbers.
+        self.canvas.bind('<MouseWheel>', self.redirect_mousewheel_event)
+
+        # Redirect mouse button events to the main editor text widget,
+        # except for the left mouse button (1).
+        #
+        # Note: X-11 sends Button-4 and Button-5 events for the scroll wheel.
+        def bind_mouse_event(event_name, target_event_name):
+            handler = functools.partial(self.redirect_mousebutton_event,
+                                        event_name=target_event_name)
+            self.canvas.bind(event_name, handler)
+
+        for button in [2, 3, 4, 5]:
+            for event_name in (f'<Button-{button}>',
+                               f'<ButtonRelease-{button}>',
+                               f'<B{button}-Motion>',
+                               ):
+                bind_mouse_event(event_name, target_event_name=event_name)
+
+            # Convert double- and triple-click events to normal click events,
+            # since event_generate() doesn't allow generating such events.
+            for event_name in (f'<Double-Button-{button}>',
+                               f'<Triple-Button-{button}>',
+                               ):
+                bind_mouse_event(event_name,
+                                 target_event_name=f'<Button-{button}>')
+
+
 def _linenumbers_drag_scrolling(parent):  # htest #
     from idlelib.idle_test.test_sidebar import Dummy_editwin
 

Lib/idlelib/sidebar.py

@@ -160,8 +160,10 @@ def expand(self, event=None):
             if not confirm:
                 return "break"
 
-        self.base_text.insert(self.text.index(self), self.s, self.tags)
+        index = self.text.index(self)
+        self.base_text.insert(index, self.s, self.tags)
         self.base_text.delete(self)
+        self.editwin.on_squeezed_expand(index, self.s, self.tags)
         self.squeezer.expandingbuttons.remove(self)
 
     def copy(self, event=None):
@@ -285,12 +287,10 @@ def count_lines(self, s):
         """
         return count_lines_with_wrapping(s, self.editwin.width)
 
-    def squeeze_current_text_event(self, event):
-        """squeeze-current-text event handler
+    def squeeze_current_text(self):
+        """Squeeze the text block where the insertion cursor is.
 
-        Squeeze the block of text inside which contains the "insert" cursor.
-
-        If the insert cursor is not in a squeezable block of text, give the
+        If the cursor is not in a squeezable block of text, give the
         user a small warning and do nothing.
         """
         # Set tag_name to the first valid tag found on the "insert" cursor.

Lib/idlelib/squeezer.py

@@ -0,0 +1 @@
+IDLE's shell now shows prompts in a separate side-bar.