perf(langserver): optimize analysing and collecting diagnostics

Author: d-biehl

packages/core/src/robotcode/core/text_document.py

@@ -215,13 +215,16 @@ def cache_invalidated(sender) -> None: ...
 
     @contextmanager
     def _cache_invalidating(self) -> Iterator[None]:
-        self.cache_invalidate()
+        send_events = len(self._cache) > 0
+        if send_events:
+            self.cache_invalidate()
         try:
             with self._lock:
                 yield
         finally:
             self._invalidate_cache()
-            self.cache_invalidated(self)
+            if send_events:
+                self.cache_invalidated(self)
 
     def _invalidate_cache(self) -> None:
         self._cache.clear()

packages/language_server/src/robotcode/language_server/common/parts/diagnostics.py

@@ -1,5 +1,4 @@
 import ast
-import threading
 from concurrent.futures import CancelledError
 from typing import TYPE_CHECKING, Any, List, Optional
 
@@ -24,6 +23,7 @@
     GlobalVariableDefinition,
     LibraryArgumentDefinition,
 )
+from robotcode.robot.diagnostics.library_doc import LibraryDoc
 from robotcode.robot.diagnostics.namespace import Namespace
 from robotcode.robot.utils.ast import (
     iter_nodes,
@@ -32,7 +32,7 @@
 )
 from robotcode.robot.utils.stubs import HasError, HasErrors, HeaderAndBodyBlock
 
-from ...common.parts.diagnostics import DiagnosticsResult
+from ...common.parts.diagnostics import DiagnosticsCollectType, DiagnosticsResult
 
 if TYPE_CHECKING:
     from ..protocol import RobotLanguageServerProtocol
@@ -58,57 +58,81 @@ def __init__(self, parent: "RobotLanguageServerProtocol") -> None:
         self.parent.diagnostics.collect.add(self.collect_unused_keyword_references)
         self.parent.diagnostics.collect.add(self.collect_unused_variable_references)
 
-        self._collect_unused_references_event = threading.Event()
-
-        self.parent.diagnostics.on_workspace_diagnostics_analyze.add(self._on_workspace_diagnostics_analyze)
-        self.parent.diagnostics.on_workspace_diagnostics_collect.add(self._on_workspace_diagnostics_collect)
+        self.parent.diagnostics.on_get_related_documents.add(self._on_get_related_documents)
 
     def _on_initialized(self, sender: Any) -> None:
         self.parent.diagnostics.analyze.add(self.analyze)
-
-        self.parent.documents_cache.namespace_invalidated.add(self.namespace_invalidated)
-
-    def _on_workspace_diagnostics_analyze(self, sender: Any) -> None:
-        self._collect_unused_references_event.clear()
-
-    def _on_workspace_diagnostics_collect(self, sender: Any) -> None:
-        self._collect_unused_references_event.set()
+        self.parent.documents_cache.namespace_invalidated.add(self._on_namespace_invalidated)
+        self.parent.documents_cache.namespace_initialized(self._on_namespace_initialized)
+        self.parent.documents_cache.libraries_changed.add(self._on_libraries_changed)
+        self.parent.documents_cache.variables_changed.add(self._on_variables_changed)
+
+    def _on_libraries_changed(self, sender: Any, libraries: List[LibraryDoc]) -> None:
+        for doc in self.parent.documents.documents:
+            namespace = self.parent.documents_cache.get_only_initialized_namespace(doc)
+            if namespace is not None:
+                lib_docs = (e.library_doc for e in namespace.get_libraries().values())
+                if any(lib_doc in lib_docs for lib_doc in libraries):
+                    self.parent.diagnostics.force_refresh_document(doc)
+
+    def _on_variables_changed(self, sender: Any, variables: List[LibraryDoc]) -> None:
+        for doc in self.parent.documents.documents:
+            namespace = self.parent.documents_cache.get_only_initialized_namespace(doc)
+            if namespace is not None:
+                lib_docs = (e.library_doc for e in namespace.get_imported_variables().values())
+                if any(lib_doc in lib_docs for lib_doc in variables):
+                    self.parent.diagnostics.force_refresh_document(doc)
 
     @language_id("robotframework")
     def analyze(self, sender: Any, document: TextDocument) -> None:
         self.parent.documents_cache.get_namespace(document).analyze()
 
     @language_id("robotframework")
-    def namespace_invalidated(self, sender: Any, namespace: Namespace) -> None:
-        self._collect_unused_references_event.clear()
+    def _on_namespace_initialized(self, sender: Any, namespace: Namespace) -> None:
+        if namespace.document is not None:
+            self.parent.diagnostics.force_refresh_document(namespace.document)
+
+    @language_id("robotframework")
+    def _on_namespace_invalidated(self, sender: Any, namespace: Namespace) -> None:
+        if namespace.document is not None:
+            namespace.document.remove_cache_entry(self._collect_model_errors)
+            namespace.document.remove_cache_entry(self._collect_token_errors)
+
+    @language_id("robotframework")
+    def _on_get_related_documents(self, sender: Any, document: TextDocument) -> Optional[List[TextDocument]]:
+        namespace = self.parent.documents_cache.get_only_initialized_namespace(document)
+        if namespace is None:
+            return None
 
-        self._namespace_invalidated(namespace)
+        result = []
 
-        self.parent.diagnostics.break_workspace_diagnostics_loop()
+        resources = namespace.get_resources().values()
+        for r in resources:
+            if r.library_doc.source:
+                doc = self.parent.documents.get(Uri.from_path(r.library_doc.source).normalized())
+                if doc is not None:
+                    result.append(doc)
 
-    def _namespace_invalidated(self, namespace: Namespace) -> None:
-        if namespace.document is not None:
-            refresh = namespace.document.opened_in_editor
+        lib_doc = namespace.get_library_doc()
+        for doc in self.parent.documents.documents:
+            if doc.language_id != "robotframework":
+                continue
 
-            self.parent.diagnostics.force_refresh_document(namespace.document, False)
+            doc_namespace = self.parent.documents_cache.get_only_initialized_namespace(doc)
+            if doc_namespace is None:
+                continue
 
-            if namespace.is_initialized():
-                resources = namespace.get_resources().values()
-                for r in resources:
-                    if r.library_doc.source:
-                        doc = self.parent.documents.get(Uri.from_path(r.library_doc.source).normalized())
-                        if doc is not None:
-                            refresh |= doc.opened_in_editor
-                            self.parent.diagnostics.force_refresh_document(doc, False)
+            if doc_namespace.is_analyzed():
+                for ref in doc_namespace.get_namespace_references():
+                    if ref.library_doc == lib_doc:
+                        result.append(doc)
 
-            if refresh:
-                self.parent.diagnostics.refresh()
+        return result
 
     @language_id("robotframework")
-    def collect_namespace_diagnostics(self, sender: Any, document: TextDocument) -> DiagnosticsResult:
-        return document.get_cache(self._collect_namespace_diagnostics)
-
-    def _collect_namespace_diagnostics(self, document: TextDocument) -> DiagnosticsResult:
+    def collect_namespace_diagnostics(
+        self, sender: Any, document: TextDocument, diagnostics_type: DiagnosticsCollectType
+    ) -> DiagnosticsResult:
         try:
             namespace = self.parent.documents_cache.get_namespace(document)
 
@@ -172,7 +196,9 @@ def _create_error_from_token(self, token: Token, source: Optional[str] = None) -
 
     @language_id("robotframework")
     @_logger.call
-    def collect_token_errors(self, sender: Any, document: TextDocument) -> DiagnosticsResult:
+    def collect_token_errors(
+        self, sender: Any, document: TextDocument, diagnostics_type: DiagnosticsCollectType
+    ) -> DiagnosticsResult:
         return document.get_cache(self._collect_token_errors)
 
     def _collect_token_errors(self, document: TextDocument) -> DiagnosticsResult:
@@ -238,7 +264,9 @@ def _collect_token_errors(self, document: TextDocument) -> DiagnosticsResult:
 
     @language_id("robotframework")
     @_logger.call
-    def collect_model_errors(self, sender: Any, document: TextDocument) -> DiagnosticsResult:
+    def collect_model_errors(
+        self, sender: Any, document: TextDocument, diagnostics_type: DiagnosticsCollectType
+    ) -> DiagnosticsResult:
         return document.get_cache(self._collect_model_errors)
 
     def _collect_model_errors(self, document: TextDocument) -> DiagnosticsResult:
@@ -284,13 +312,15 @@ def _collect_model_errors(self, document: TextDocument) -> DiagnosticsResult:
 
     @language_id("robotframework")
     @_logger.call
-    def collect_unused_keyword_references(self, sender: Any, document: TextDocument) -> DiagnosticsResult:
+    def collect_unused_keyword_references(
+        self, sender: Any, document: TextDocument, diagnostics_type: DiagnosticsCollectType
+    ) -> DiagnosticsResult:
         config = self.parent.workspace.get_configuration(AnalysisConfig, document.uri)
 
         if not config.find_unused_references:
             return DiagnosticsResult(self.collect_unused_keyword_references, [])
 
-        if not self._collect_unused_references_event.is_set():
+        if diagnostics_type != DiagnosticsCollectType.SLOW:
             return DiagnosticsResult(self.collect_unused_keyword_references, None, True)
 
         return self._collect_unused_keyword_references(document)
@@ -341,13 +371,15 @@ def _collect_unused_keyword_references(self, document: TextDocument) -> Diagnost
 
     @language_id("robotframework")
     @_logger.call
-    def collect_unused_variable_references(self, sender: Any, document: TextDocument) -> DiagnosticsResult:
+    def collect_unused_variable_references(
+        self, sender: Any, document: TextDocument, diagnostics_type: DiagnosticsCollectType
+    ) -> DiagnosticsResult:
         config = self.parent.workspace.get_configuration(AnalysisConfig, document.uri)
 
         if not config.find_unused_references:
             return DiagnosticsResult(self.collect_unused_variable_references, [])
 
-        if not self._collect_unused_references_event.is_set():
+        if diagnostics_type != DiagnosticsCollectType.SLOW:
             return DiagnosticsResult(self.collect_unused_variable_references, None, True)
 
         return self._collect_unused_variable_references(document)

packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py

@@ -66,7 +66,7 @@ def __init__(self, parent: "RobotLanguageServerProtocol") -> None:
 
         parent.references.collect.add(self.collect)
         parent.documents.did_change.add(self.document_did_change)
-
+        parent.documents.on_document_cache_invalidated(self.document_did_change)
         parent.diagnostics.on_workspace_diagnostics_break.add(self.on_workspace_diagnostics_break)
 
     @event

packages/language_server/src/robotcode/language_server/robotframework/parts/references.py

@@ -14,7 +14,7 @@
 from robotcode.core.utils.version import Version, create_version_from_str
 from robotcode.core.workspace import WorkspaceFolder
 
-from ...common.parts.diagnostics import DiagnosticsResult
+from ...common.parts.diagnostics import DiagnosticsCollectType, DiagnosticsResult
 from ..configuration import RoboCopConfig
 from .protocol_part import RobotLanguageServerProtocolPart
 
@@ -50,7 +50,9 @@ def get_config(self, document: TextDocument) -> Optional[RoboCopConfig]:
 
     @language_id("robotframework")
     @_logger.call
-    def collect_diagnostics(self, sender: Any, document: TextDocument) -> DiagnosticsResult:
+    def collect_diagnostics(
+        self, sender: Any, document: TextDocument, diagnostics_type: DiagnosticsCollectType
+    ) -> DiagnosticsResult:
         workspace_folder = self.parent.workspace.get_workspace_folder(document.uri)
         if workspace_folder is not None:
             extension_config = self.get_config(document)

packages/language_server/src/robotcode/language_server/robotframework/parts/robocop_diagnostics.py

@@ -22,7 +22,6 @@
 from robotcode.core.documents_manager import DocumentsManager
 from robotcode.core.event import event
 from robotcode.core.filewatcher import FileWatcherManagerBase
-from robotcode.core.language import language_id_filter
 from robotcode.core.text_document import TextDocument
 from robotcode.core.uri import Uri
 from robotcode.core.utils.logging import LoggingDescriptor
@@ -32,6 +31,7 @@
 from ..utils import get_robot_version
 from ..utils.stubs import Languages
 from .imports_manager import ImportsManager
+from .library_doc import LibraryDoc
 from .namespace import DocumentType, Namespace
 from .workspace_config import AnalysisRobotConfig, CacheConfig, RobotConfig
 
@@ -421,22 +421,29 @@ def get_general_namespace(self, document: TextDocument) -> Namespace:
     def __get_general_namespace(self, document: TextDocument) -> Namespace:
         return self.__get_namespace_for_document_type(document, DocumentType.GENERAL)
 
+    @event
+    def namespace_initialized(sender, namespace: Namespace) -> None: ...
+
     @event
     def namespace_invalidated(sender, namespace: Namespace) -> None: ...
 
     def __invalidate_namespace(self, sender: Namespace) -> None:
         document = sender.document
         if document is not None:
-            document.invalidate_cache()
+            document.remove_cache_entry(self.__get_general_namespace)
+            document.remove_cache_entry(self.__get_init_namespace)
+            document.remove_cache_entry(self.__get_resource_namespace)
+            document.remove_cache_entry(self.__get_namespace)
 
-            self.namespace_invalidated(self, sender, callback_filter=language_id_filter(document))
+            self.namespace_invalidated(self, sender)
 
     def __namespace_initialized(self, sender: Namespace) -> None:
         if sender.document is not None:
             self._logger.debug(
                 lambda: f"Save initialized Namespace: {sender.document.uri if sender.document else None}"
             )
             sender.document.set_data(self.INITIALIZED_NAMESPACE, sender)
+            self.namespace_initialized(self, sender)
 
     def get_initialized_namespace(self, document: TextDocument) -> Namespace:
         result: Optional[Namespace] = document.get_data(self.INITIALIZED_NAMESPACE)
@@ -445,6 +452,11 @@ def get_initialized_namespace(self, document: TextDocument) -> Namespace:
             result = self.get_namespace(document)
         return result
 
+    def get_only_initialized_namespace(self, document: TextDocument) -> Optional[Namespace]:
+        result: Optional[Namespace] = document.get_data(self.INITIALIZED_NAMESPACE)
+
+        return result
+
     def __get_namespace_for_document_type(
         self, document: TextDocument, document_type: Optional[DocumentType]
     ) -> Namespace:
@@ -495,7 +507,7 @@ def create_imports_manager(self, root_uri: Uri) -> ImportsManager:
         ]
 
         analysis_config = self.workspace.get_configuration(AnalysisRobotConfig, root_uri)
-        return ImportsManager(
+        result = ImportsManager(
             self.documents_manager,
             self.file_watcher_manager,
             self,
@@ -510,6 +522,30 @@ def create_imports_manager(self, root_uri: Uri) -> ImportsManager:
             cache_base_path,
         )
 
+        result.libraries_changed.add(self._on_libraries_changed)
+        result.resources_changed.add(self._on_resources_changed)
+        result.variables_changed.add(self._on_variables_changed)
+
+        return result
+
+    @event
+    def libraries_changed(sender, libraries: List[LibraryDoc]) -> None: ...
+
+    @event
+    def resources_changed(sender, resources: List[LibraryDoc]) -> None: ...
+
+    @event
+    def variables_changed(sender, variables: List[LibraryDoc]) -> None: ...
+
+    def _on_libraries_changed(self, sender: ImportsManager, libraries: List[LibraryDoc]) -> None:
+        self.libraries_changed(self, libraries)
+
+    def _on_resources_changed(self, sender: ImportsManager, resources: List[LibraryDoc]) -> None:
+        self.resources_changed(self, resources)
+
+    def _on_variables_changed(self, sender: ImportsManager, variables: List[LibraryDoc]) -> None:
+        self.variables_changed(self, variables)
+
     def default_imports_manager(self) -> ImportsManager:
         with self._imports_managers_lock:
             if self._default_imports_manager is None:

packages/robot/src/robotcode/robot/diagnostics/document_cache_helper.py

@@ -195,11 +195,10 @@ class VariableDefinition(SourceEntity):
     value: Any = field(default=None, compare=False)
     value_is_native: bool = field(default=False, compare=False)
 
-    @property
-    def matcher(self) -> VariableMatcher:
-        if not hasattr(self, "__matcher"):
-            self.__matcher = VariableMatcher(self.name)
-        return self.__matcher
+    matcher: VariableMatcher = field(init=False, compare=False)
+
+    def __post_init__(self) -> None:
+        self.matcher = VariableMatcher(self.name)
 
     @single_call
     def __hash__(self) -> int:

packages/robot/src/robotcode/robot/diagnostics/entities.py

@@ -503,8 +503,8 @@ def __init__(
         super().__init__()
 
         self.documents_manager = documents_manager
-        self.documents_manager.did_create_uri.add(self.possible_imports_modified)
-        self.documents_manager.did_change.add(self.possible_resource_document_modified)
+        self.documents_manager.did_create_uri.add(self._on_possible_imports_modified)
+        self.documents_manager.did_change.add(self._on_possible_resource_document_modified)
 
         self.file_watcher_manager: FileWatcherManagerBase = (
             file_watcher_manager if file_watcher_manager is not None else FileWatcherManagerDummy()
@@ -685,12 +685,15 @@ def variables_changed(sender, variables: List[LibraryDoc]) -> None: ...
     @event
     def imports_changed(sender, uri: DocumentUri) -> None: ...
 
-    def possible_imports_modified(self, sender: Any, uri: DocumentUri) -> None:
+    def _on_possible_imports_modified(self, sender: Any, uri: DocumentUri) -> None:
         # TODO: do we really need this?
         self.imports_changed(self, uri)
 
     @language_id("robotframework")
-    def possible_resource_document_modified(self, sender: Any, document: TextDocument) -> None:
+    def _on_possible_resource_document_modified(self, sender: Any, document: TextDocument) -> None:
+        run_as_task(self.__on_possible_resource_document_modified, sender, document)
+
+    def __on_possible_resource_document_modified(self, sender: Any, document: TextDocument) -> None:
         with self._resource_document_changed_timer_lock:
             if document in self._resource_document_changed_documents:
                 return
@@ -714,7 +717,7 @@ def __resource_documents_changed(self) -> None:
             self._resource_document_changed_documents = set()
 
         for document in documents:
-            run_as_task(self.__resource_document_changed, document).result()
+            self.__resource_document_changed(document)
 
     def __resource_document_changed(self, document: TextDocument) -> None:
         resource_changed: List[LibraryDoc] = []