PR: Make Pygments to work correctly with QSyntaxHighlighter

spyder/utils/syntaxhighlighters.py

@@ -27,6 +27,7 @@
 from spyder.config.main import CONF
 from spyder.py3compat import builtins, is_text_string, to_text_string
 from spyder.utils.sourcecode import CELL_LANGUAGES
+from spyder.utils.workers import WorkerManager
 
 
 PYGMENTS_REQVER = '>=2.0'
@@ -63,7 +64,6 @@
 CUSTOM_EXTENSION_LEXER = {'.ipynb': 'json',
                           '.txt': 'text',
                           '.nt': 'bat',
-                          '.scss': 'css',
                           '.m': 'matlab',
                           ('.properties', '.session', '.inf', '.reg', '.url',
                            '.cfg', '.cnf', '.aut', '.iss'): 'ini'}
@@ -1052,6 +1052,7 @@ class PygmentsSH(BaseSH):
     # Store the language name and a ref to the lexer
     _lang_name = None
     _lexer = None
+    _charlist = []
     # Syntax highlighting states (from one text block to another):
     NORMAL = 0
     def __init__(self, parent, font=None, color_scheme=None):
@@ -1074,33 +1075,81 @@ def __init__(self, parent, font=None, color_scheme=None):
         # Load Pygments' Lexer
         if self._lang_name is not None:
             self._lexer = get_lexer_by_name(self._lang_name)
+
+
+
+        # Connect document updates to a re-lexing of the document.
+        # parent.contentsChange.connect(self._make_charlist)
         BaseSH.__init__(self, parent, font, color_scheme)
+        self._worker_manager = WorkerManager()
+        self._charlist = []
+        self._allow_highlight = True
+
+    def make_charlist(self):
+        """"""
+        def test_output(worker, output, error):
+            self._charlist = output
+            if error is None and output:
+                self._allow_highlight = True
+                self.rehighlight()
+            self._allow_highlight = False
+
+        text = to_text_string(self.document().toPlainText())
+        tokens = self._lexer.get_tokens(text)
+        self._worker_manager.terminate_all()
+        worker = self._worker_manager.create_python_worker(
+            self._make_charlist,
+            tokens,
+            self._tokmap,
+            self.formats,
+        )
+        worker.sig_finished.connect(test_output)
+        worker.start()
+
+    def _make_charlist(self, tokens, tokmap, formats):
+        """
+        Parses the complete text and stores format for each character.
+
+        Uses the attached lexer to parse into a list of tokens and Pygments
+        token types.  Then breaks tokens into individual letters, each with a
+        Spyder token type attached.  Stores this list as self._charlist.
+
+        It's attached to the contentsChange signal of the parent QTextDocument
+        so that the charlist is updated whenever the document changes.
+        """
 
-    def get_fmt(self, typ):
-        """ Get the format code for this type """
-        # Exact matches first
-        for key in self._tokmap:
-            if typ is key:
-                return self._tokmap[key]            
-        # Partial (parent-> child) matches
-        for key in self._tokmap:
-            if typ in key.subtypes:
-                return self._tokmap[key]
-        return 'normal'
+        def _get_fmt(typ):
+            """Get the Spyder format code for the given Pygments token type."""
+            # Exact matches first
+            if typ in tokmap:
+                return tokmap[typ]
+            # Partial (parent-> child) matches
+            for key, val in tokmap.items():
+                if typ in key: # Checks if typ is a subtype of key.
+                    return val
+
+            return 'normal'
+
+        charlist = []
+        for typ, token in tokens:
+            fmt = formats[_get_fmt(typ)]
+            for letter in token:
+                charlist.append((fmt, letter))
+
+        return charlist
 
     def highlightBlock(self, text):
-        """ Actually highlight the block """        
-        text = to_text_string(text)                
-        lextree = self._lexer.get_tokens(text)        
-        ct = 0
-        for item in lextree:            
-            typ, val = item            
-            key = self.get_fmt(typ)
-            start = ct
-            ct += len(val)        
-            self.setFormat(start, ct-start, self.formats[key])
-
-        self.highlight_spaces(text)
+        """ Actually highlight the block"""
+        # Note that an undefined blockstate is equal to -1, so the first block
+        # will have the correct behaviour of starting at 0.
+        if self._allow_highlight:
+            start = self.previousBlockState() + 1
+            end = start + len(text)
+            for i, (fmt, letter) in enumerate(self._charlist[start:end]):
+                self.setFormat(i, 1, fmt)
+            self.setCurrentBlockState(end)
+            self.highlight_spaces(text)
+
 
 def guess_pygments_highlighter(filename):
     """Factory to generate syntax highlighter for the given filename.