perf(langserver): speedup semantic highlightning a lot

Author: d-biehl

package.json

@@ -241,6 +241,9 @@
         "rules": {
           "*.documentation:robotframework": {
             "fontStyle": "italic"
+          },
+          "*.embedded:robotframework": {
+            "fontStyle": "italic"
           }
         }
       }
@@ -249,6 +252,10 @@
       {
         "id": "builtin",
         "description": "built in library, keyword or variable"
+      },
+      {
+        "id": "embedded",
+        "description": "embedded argument"
       }
     ],
     "semanticTokenScopes": [
@@ -1650,4 +1657,4 @@
   "workspaces": [
     "docs"
   ]
-}
+}

packages/core/src/robotcode/core/text_document.py

@@ -1,4 +1,5 @@
 import collections
+import functools
 import inspect
 import io
 import weakref
@@ -47,10 +48,18 @@ def position_from_utf16(lines: List[str], position: Position) -> Position:
     return Position(line=position.line, character=utf32_offset)
 
 
+@functools.lru_cache(maxsize=2048)
+def has_multibyte_char(line: str) -> bool:
+    return any(is_multibyte_char(c) for c in line)
+
+
 def position_to_utf16(lines: List[str], position: Position) -> Position:
     if position.line >= len(lines):
         return position
 
+    if not has_multibyte_char(lines[position.line]):
+        return position
+
     utf16_counter = 0
 
     for i, c in enumerate(lines[position.line]):

packages/core/src/robotcode/core/utils/logging.py

@@ -258,7 +258,10 @@ def measure_time(
                     self._measure_contexts[context_name] = depth
 
                 self._log_measure_time(
-                    level, f"{'  '*depth}End {msg() if callable(msg) else msg} took {duration} seconds", *args, **kwargs
+                    level,
+                    f"{'  '*depth}End {msg() if callable(msg) else msg} took {duration} seconds",
+                    *args,
+                    **kwargs,
                 )
         else:
             yield

packages/language_server/src/robotcode/language_server/robotframework/parts/semantic_tokens.py

@@ -23,6 +23,7 @@
 from robot.parsing.lexer.tokens import Token
 from robot.parsing.model.statements import (
     Arguments,
+    Documentation,
     Fixture,
     KeywordCall,
     LibraryImport,
@@ -62,6 +63,7 @@
 from robotcode.robot.diagnostics.namespace import DEFAULT_BDD_PREFIXES, Namespace
 from robotcode.robot.utils import get_robot_version
 from robotcode.robot.utils.ast import (
+    cached_isinstance,
     iter_nodes,
     iter_over_keyword_names_and_owners,
     token_in_range,
@@ -120,6 +122,7 @@ class RobotSemTokenTypes(Enum):
 
 class RobotSemTokenModifiers(Enum):
     BUILTIN = "builtin"
+    EMBEDDED = "embedded"
 
 
 @dataclass
@@ -340,6 +343,7 @@ def generate_sem_sub_tokens(
         node: ast.AST,
         col_offset: Optional[int] = None,
         length: Optional[int] = None,
+        yield_arguments: bool = False,
     ) -> Iterator[SemTokenInfo]:
         sem_info = cls.mapping().get(token.type, None) if token.type is not None else None
         if sem_info is not None:
@@ -391,7 +395,7 @@ def generate_sem_sub_tokens(
                     yield SemTokenInfo.from_token(token, sem_type, sem_mod)
 
             elif token.type in [Token.KEYWORD, ROBOT_KEYWORD_INNER] or (
-                token.type == Token.NAME and isinstance(node, (Fixture, Template, TestTemplate))
+                token.type == Token.NAME and cached_isinstance(node, Fixture, Template, TestTemplate)
             ):
                 if (
                     namespace.find_keyword(
@@ -461,6 +465,9 @@ def generate_sem_sub_tokens(
 
                 kw_index = len(kw_namespace) + 1 if kw_namespace else 0
 
+                if token.type == Token.NAME and kw_doc is not None:
+                    sem_type = RobotSemTokenTypes.KEYWORD
+
                 if kw_namespace:
                     kw = token.value[kw_index:]
 
@@ -501,13 +508,25 @@ def generate_sem_sub_tokens(
                                 col_offset + kw_index + start,
                                 arg_start - start,
                             )
-                            yield SemTokenInfo.from_token(
-                                token,
-                                RobotSemTokenTypes.EMBEDDED_ARGUMENT,
-                                sem_mod,
-                                col_offset + kw_index + arg_start,
-                                arg_end - arg_start,
+
+                            embedded_token = Token(
+                                Token.ARGUMENT,
+                                token.value[arg_start:arg_end],
+                                token.lineno,
+                                token.col_offset + arg_start,
                             )
+
+                            for sub_token in ModelHelper.tokenize_variables(
+                                embedded_token,
+                                ignore_errors=True,
+                                identifiers="$@&%",
+                            ):
+                                for e in cls.generate_sem_sub_tokens(
+                                    namespace, builtin_library_doc, sub_token, node, yield_arguments=True
+                                ):
+                                    e.sem_modifiers = {RobotSemTokenModifiers.EMBEDDED}
+                                    yield e
+
                             start = arg_end + 1
 
                         if start < end:
@@ -521,7 +540,7 @@ def generate_sem_sub_tokens(
 
                 else:
                     yield SemTokenInfo.from_token(token, sem_type, sem_mod, col_offset + kw_index, len(kw))
-            elif token.type == Token.NAME and isinstance(node, (LibraryImport, ResourceImport, VariablesImport)):
+            elif token.type == Token.NAME and cached_isinstance(node, LibraryImport, ResourceImport, VariablesImport):
                 if "\\" in token.value:
                     if col_offset is None:
                         col_offset = token.col_offset
@@ -543,7 +562,9 @@ def generate_sem_sub_tokens(
                         length,
                     )
             elif get_robot_version() >= (5, 0) and token.type == Token.OPTION:
-                if (isinstance(node, ExceptHeader) or isinstance(node, WhileHeader)) and "=" in token.value:
+                if (
+                    cached_isinstance(node, ExceptHeader) or cached_isinstance(node, WhileHeader)
+                ) and "=" in token.value:
                     if col_offset is None:
                         col_offset = token.col_offset
 
@@ -589,7 +610,12 @@ def generate_sem_sub_tokens(
                     1,
                 )
             else:
-                if token.type != Token.ARGUMENT or token.type != Token.NAME and isinstance(node, Metadata):
+                if (
+                    yield_arguments
+                    or token.type != Token.ARGUMENT
+                    or token.type != Token.NAME
+                    and cached_isinstance(node, Metadata)
+                ):
                     yield SemTokenInfo.from_token(token, sem_type, sem_mod, col_offset, length)
 
     def generate_sem_tokens(
@@ -602,25 +628,25 @@ def generate_sem_tokens(
         if (
             token.type in {Token.ARGUMENT, Token.TESTCASE_NAME, Token.KEYWORD_NAME}
             or token.type == Token.NAME
-            and isinstance(node, (VariablesImport, LibraryImport, ResourceImport))
+            and cached_isinstance(node, VariablesImport, LibraryImport, ResourceImport)
         ):
-            if (isinstance(node, Variable) and token.type == Token.ARGUMENT and node.name and node.name[0] == "&") or (
-                isinstance(node, Arguments)
-            ):
+            if (
+                cached_isinstance(node, Variable) and token.type == Token.ARGUMENT and node.name and node.name[0] == "&"
+            ) or (cached_isinstance(node, Arguments)):
                 name, value = split_from_equals(token.value)
                 if value is not None:
                     length = len(name)
 
                     yield SemTokenInfo.from_token(
                         Token(
-                            ROBOT_NAMED_ARGUMENT if isinstance(node, Variable) else SemanticTokenTypes.PARAMETER,
+                            ROBOT_NAMED_ARGUMENT if cached_isinstance(node, Variable) else SemanticTokenTypes.PARAMETER,
                             name,
                             token.lineno,
                             token.col_offset,
                         ),
                         (
                             RobotSemTokenTypes.NAMED_ARGUMENT
-                            if isinstance(node, Variable)
+                            if cached_isinstance(node, Variable)
                             else SemanticTokenTypes.PARAMETER
                         ),
                     )
@@ -640,7 +666,7 @@ def generate_sem_tokens(
                         token.col_offset + length + 1,
                         token.error,
                     )
-                elif isinstance(node, Arguments) and name:
+                elif cached_isinstance(node, Arguments) and name:
                     yield SemTokenInfo.from_token(
                         Token(
                             ROBOT_NAMED_ARGUMENT,
@@ -663,11 +689,13 @@ def generate_sem_tokens(
                 ignore_errors=True,
                 identifiers="$" if token.type == Token.KEYWORD_NAME else "$@&%",
             ):
-                for e in self.generate_sem_sub_tokens(namespace, builtin_library_doc, sub_token, node):
+                for e in self.generate_sem_sub_tokens(
+                    namespace, builtin_library_doc, sub_token, node, yield_arguments=True
+                ):
                     yield e
 
         else:
-            for e in self.generate_sem_sub_tokens(namespace, builtin_library_doc, token, node):
+            for e in self.generate_sem_sub_tokens(namespace, builtin_library_doc, token, node, yield_arguments=True):
                 yield e
 
     def generate_run_kw_tokens(
@@ -956,8 +984,8 @@ def get_tokens() -> Iterator[Tuple[Token, ast.AST]]:
             for node in iter_nodes(model):
                 check_current_task_canceled()
 
-                if isinstance(node, Statement):
-                    if isinstance(node, LibraryImport) and node.name:
+                if cached_isinstance(node, Statement):
+                    if cached_isinstance(node, LibraryImport) and node.name:
                         lib_doc = namespace.get_imported_library_libdoc(node.name, node.args, node.alias)
                         kw_doc = lib_doc.inits.keywords[0] if lib_doc and lib_doc.inits else None
                         if lib_doc is not None:
@@ -1009,7 +1037,7 @@ def get_tokens() -> Iterator[Tuple[Token, ast.AST]]:
 
                                 yield token, node
                             continue
-                    if isinstance(node, VariablesImport) and node.name:
+                    if cached_isinstance(node, VariablesImport) and node.name:
                         lib_doc = namespace.get_imported_variables_libdoc(node.name, node.args)
                         kw_doc = lib_doc.inits.keywords[0] if lib_doc and lib_doc.inits else None
                         if lib_doc is not None:
@@ -1061,12 +1089,12 @@ def get_tokens() -> Iterator[Tuple[Token, ast.AST]]:
 
                                 yield token, node
                             continue
-                    if isinstance(node, (KeywordCall, Fixture)):
+                    if cached_isinstance(node, KeywordCall, Fixture):
                         kw_token = cast(
                             Token,
                             (
                                 node.get_token(Token.KEYWORD)
-                                if isinstance(node, KeywordCall)
+                                if cached_isinstance(node, KeywordCall)
                                 else node.get_token(Token.NAME)
                             ),
                         )
@@ -1109,8 +1137,16 @@ def get_tokens() -> Iterator[Tuple[Token, ast.AST]]:
                                     yield kw_res
 
                                 continue
+                    if cached_isinstance(node, Documentation):
+                        for token in node.tokens:
+                            if token.type == Token.ARGUMENT:
+                                continue
+                            yield token, node
+                        continue
 
                     for token in node.tokens:
+                        if token.type == Token.COMMENT:
+                            continue
                         yield token, node
 
         lines = document.get_lines()
@@ -1136,6 +1172,7 @@ def get_tokens() -> Iterator[Tuple[Token, ast.AST]]:
                         ),
                     ),
                 )
+
                 token_col_offset = token_range.start.character
                 token_length = token_range.end.character - token_range.start.character
 

packages/robot/src/robotcode/robot/utils/ast.py

@@ -2,7 +2,9 @@
 
 import ast
 import itertools
-from typing import Any, Iterator, List, Optional, Sequence, Set, Tuple
+from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple, Type, TypeVar, Union
+
+from typing_extensions import TypeGuard
 
 from robot.errors import VariableError
 from robot.parsing.lexer.tokens import Token
@@ -17,17 +19,42 @@
 else:
     from robot.variables.search import VariableMatches as VariableIterator
 
+_cached_isinstance_cache: Dict[Tuple[type, Tuple[type, ...]], bool] = {}
+
+_T = TypeVar("_T")
+
+
+def cached_isinstance(obj: Any, *expected_types: Type[_T]) -> TypeGuard[Union[_T]]:
+    try:
+        t = type(obj)
+        if (t, expected_types) in _cached_isinstance_cache:
+            return _cached_isinstance_cache[(t, expected_types)]
+
+        _cached_isinstance_cache[(t, expected_types)] = result = isinstance(obj, expected_types)
+
+        return result
+
+    except TypeError:
+        return False
+
+
+# def cached_isinstance(obj: Any, *expected_types: type) -> bool:
+#     try:
+#         return isinstance(obj, expected_types)
+#     except TypeError:
+#         return False
+
 
 def iter_nodes(node: ast.AST, descendants: bool = True) -> Iterator[ast.AST]:
     for _field, value in ast.iter_fields(node):
-        if isinstance(value, list):
+        if cached_isinstance(value, list):
             for item in value:
-                if isinstance(item, ast.AST):
+                if cached_isinstance(item, ast.AST):
                     yield item
                     if descendants:
                         yield from iter_nodes(item)
 
-        elif isinstance(value, ast.AST):
+        elif cached_isinstance(value, ast.AST):
             yield value
             if descendants:
                 yield from iter_nodes(value)
@@ -53,7 +80,7 @@ def find_from(cls, model: ast.AST) -> Tuple[Optional[ast.AST], Optional[ast.AST]
         return finder.first_statement, finder.last_statement
 
     def visit_Statement(self, statement: ast.AST) -> None:  # noqa: N802
-        if not isinstance(statement, EmptyLine):
+        if not cached_isinstance(statement, EmptyLine):
             if self.first_statement is None:
                 self.first_statement = statement
 
@@ -63,7 +90,7 @@ def visit_Statement(self, statement: ast.AST) -> None:  # noqa: N802
 def _get_non_data_range_from_node(
     node: ast.AST, only_start: bool = False, allow_comments: bool = False
 ) -> Optional[Range]:
-    if isinstance(node, Statement) and node.tokens:
+    if cached_isinstance(node, Statement) and node.tokens:
         start_token = next(
             (
                 v
@@ -115,7 +142,7 @@ def range_from_node(
     allow_comments: bool = False,
 ) -> Range:
     if skip_non_data:
-        if isinstance(node, Statement) and node.tokens:
+        if cached_isinstance(node, Statement) and node.tokens:
             result = _get_non_data_range_from_node(node, only_start, allow_comments)
             if result is not None:
                 return result

pyproject.toml

@@ -142,7 +142,7 @@ filterwarnings = "ignore:.*Using or importing the ABCs from 'collections' instea
 testpaths = ["tests"]
 junit_suite_name = "robotcode"
 # console_output_style = "classic"
-# log_cli = true
+log_cli = true
 # log_cli_level = 4
 # log_cli_format = "%(levelname)s %(name)s: %(message)s"
 asyncio_mode = "auto"