feat: Improved Handling of UTF-16 encoded multibyte characters, e.g. emojis are now handled correctly

Author: d-biehl

packages/language_server/src/robotcode/language_server/common/parts/code_action.py

@@ -75,10 +75,16 @@ async def _text_document_code_action(
         if document is None:
             return None
 
+        for c in context.diagnostics:
+            c.range = document.range_from_utf16(c.range)
+            if c.related_information is not None:
+                for r in c.related_information:
+                    r.location.range = document.range_from_utf16(r.location.range)
+
         for result in await self.collect(
             self,
             document,
-            range,
+            document.range_from_utf16(range),
             context,
             callback_filter=language_id_filter(document),
         ):
@@ -89,10 +95,10 @@ async def _text_document_code_action(
                 if result is not None:
                     results.extend(result)
 
-        if len(results) > 0:
-            return results
+        if not results:
+            return None
 
-        return None
+        return results
 
     @rpc_method(name="textDocument/codeAction/resolve", param_type=CodeAction)
     @threaded()

packages/language_server/src/robotcode/language_server/common/parts/code_lens.py

@@ -61,10 +61,13 @@ async def _text_document_code_lens(
                 if result is not None:
                     results.extend(result)
 
-        if len(results) > 0:
-            return results
+        if not results:
+            return None
+
+        for result in results:
+            result.range = document.range_to_utf16(result.range)
 
-        return None
+        return results
 
     @rpc_method(name="codeLens/resolve", param_type=CodeLens)
     @threaded()

packages/language_server/src/robotcode/language_server/common/parts/completion.py

@@ -15,9 +15,11 @@
     CompletionList,
     CompletionOptions,
     CompletionParams,
+    InsertReplaceEdit,
     Position,
     ServerCapabilities,
     TextDocumentIdentifier,
+    TextEdit,
 )
 from robotcode.language_server.common.parts.protocol_part import LanguageServerProtocolPart
 from robotcode.language_server.common.text_document import TextDocument
@@ -89,7 +91,7 @@ async def _text_document_completion(
         for result in await self.collect(
             self,
             document,
-            position,
+            document.position_from_utf16(position),
             context,
             callback_filter=language_id_filter(document),
         ):
@@ -100,23 +102,41 @@ async def _text_document_completion(
                 if result is not None:
                     results.append(result)
 
-        if len(results) > 0:
-            if any(e for e in results if isinstance(e, CompletionList)):
-                result = CompletionList(
-                    is_incomplete=any(e for e in results if isinstance(e, CompletionList) and e.is_incomplete),
-                    items=list(chain(*[r.items if isinstance(r, CompletionList) else r for r in results])),
-                )
-                if len(result.items) == 0:
-                    return None
-                return result
+        if not results:
+            return None
 
-            result = list(chain(*[k for k in results if isinstance(k, list)]))
-            if len(result) == 0:
+        for result in results:
+            if isinstance(result, CompletionList):
+                for item in result.items:
+                    if item.text_edit is not None:
+                        self.update_completion_item_to_utf16(document, item)
+
+            elif isinstance(result, list):
+                for item in result:
+                    if item.text_edit is not None:
+                        self.update_completion_item_to_utf16(document, item)
+
+        if any(e for e in results if isinstance(e, CompletionList)):
+            result = CompletionList(
+                is_incomplete=any(e for e in results if isinstance(e, CompletionList) and e.is_incomplete),
+                items=list(chain(*[r.items if isinstance(r, CompletionList) else r for r in results])),
+            )
+            if len(result.items) == 0:
                 return None
-
             return result
 
-        return None
+        result = list(chain(*[k for k in results if isinstance(k, list)]))
+        if not result:
+            return None
+
+        return result
+
+    def update_completion_item_to_utf16(self, document: TextDocument, item: CompletionItem) -> None:
+        if isinstance(item.text_edit, TextEdit):
+            item.text_edit.range = document.range_to_utf16(item.text_edit.range)
+        elif isinstance(item.text_edit, InsertReplaceEdit):
+            item.text_edit.insert = document.range_to_utf16(item.text_edit.insert)
+            item.text_edit.replace = document.range_to_utf16(item.text_edit.replace)
 
     @rpc_method(name="completionItem/resolve", param_type=CompletionItem)
     @threaded()
@@ -136,10 +156,11 @@ async def _completion_item_resolve(
                 if result is not None:
                     results.append(result)
 
-        if len(results) > 0:
-            if len(results) > 1:
-                self._logger.warning("More then one resolve result. Use the last one.")
+        if not results:
+            return params
 
-            return results[-1]
+        if len(results) > 1:
+            self._logger.warning("More then one resolve result. Use the last one.")
+        result = results[-1]
 
-        return params
+        return result

packages/language_server/src/robotcode/language_server/common/parts/declaration.py

@@ -79,9 +79,25 @@ async def _text_document_declaration(
                                 locations.append(e)
                             elif isinstance(e, LocationLink):
                                 location_links.append(e)
+
         if len(locations) == 0 and len(location_links) == 0:
             return None
 
+        if locations:
+            for location in locations:
+                doc = await self.parent.documents.get(location.uri)
+                if doc is not None:
+                    location.range = doc.range_to_utf16(location.range)
+
+        if location_links:
+            for location_link in location_links:
+                doc = await self.parent.documents.get(location_link.target_uri)
+                if doc is not None:
+                    location_link.target_range = doc.range_to_utf16(location_link.target_range)
+                    location_link.target_selection_range = doc.range_to_utf16(location_link.target_selection_range)
+                if location_link.origin_selection_range is not None:
+                    location_link.origin_selection_range = document.range_to_utf16(location_link.origin_selection_range)
+
         if len(locations) > 0 and len(location_links) == 0:
             if len(locations) == 1:
                 return locations[0]

packages/language_server/src/robotcode/language_server/common/parts/definition.py

@@ -59,7 +59,9 @@ async def _text_document_definition(
         if document is None:
             return None
 
-        for result in await self.collect(self, document, position, callback_filter=language_id_filter(document)):
+        for result in await self.collect(
+            self, document, document.position_from_utf16(position), callback_filter=language_id_filter(document)
+        ):
             if isinstance(result, BaseException):
                 if not isinstance(result, CancelledError):
                     self._logger.exception(result, exc_info=result)
@@ -73,9 +75,25 @@ async def _text_document_definition(
                                 locations.append(e)
                             elif isinstance(e, LocationLink):
                                 location_links.append(e)
+
         if len(locations) == 0 and len(location_links) == 0:
             return None
 
+        if locations:
+            for location in locations:
+                doc = await self.parent.documents.get(location.uri)
+                if doc is not None:
+                    location.range = doc.range_to_utf16(location.range)
+
+        if location_links:
+            for location_link in location_links:
+                doc = await self.parent.documents.get(location_link.target_uri)
+                if doc is not None:
+                    location_link.target_range = doc.range_to_utf16(location_link.target_range)
+                    location_link.target_selection_range = doc.range_to_utf16(location_link.target_selection_range)
+                if location_link.origin_selection_range is not None:
+                    location_link.origin_selection_range = document.range_to_utf16(location_link.origin_selection_range)
+
         if len(locations) > 0 and len(location_links) == 0:
             if len(locations) == 1:
                 return locations[0]

packages/language_server/src/robotcode/language_server/common/parts/diagnostics.py

@@ -457,6 +457,16 @@ async def _get_diagnostics_for_document(
                 result = cast(DiagnosticsResult, result_any)
 
                 data.id = str(uuid.uuid4())
+
+                if result.diagnostics is not None:
+                    for d in result.diagnostics:
+                        d.range = document.range_to_utf16(d.range)
+
+                        for r in d.related_information or []:
+                            doc = await self.parent.documents.get(r.location.uri)
+                            if doc is not None:
+                                r.location.range = doc.range_to_utf16(r.location.range)
+
                 data.entries[result.key] = result.diagnostics
                 if result.diagnostics is not None:
                     collected_keys.append(result.key)

packages/language_server/src/robotcode/language_server/common/parts/document_highlight.py

@@ -54,7 +54,9 @@ async def _text_document_document_highlight(
         if document is None:
             return None
 
-        for result in await self.collect(self, document, position, callback_filter=language_id_filter(document)):
+        for result in await self.collect(
+            self, document, document.position_from_utf16(position), callback_filter=language_id_filter(document)
+        ):
             if isinstance(result, BaseException):
                 if not isinstance(result, CancelledError):
                     self._logger.exception(result, exc_info=result)
@@ -65,4 +67,7 @@ async def _text_document_document_highlight(
         if len(highlights) == 0:
             return None
 
+        for highlight in highlights:
+            highlight.range = document.range_to_utf16(highlight.range)
+
         return highlights

packages/language_server/src/robotcode/language_server/common/parts/document_symbols.py

@@ -104,7 +104,7 @@ async def _text_document_symbol(
         symbol_informations: List[SymbolInformation] = []
 
         document = await self.parent.documents.get(text_document.uri)
-        if not document:
+        if document is None:
             return None
 
         for result in await self.collect(self, document, callback_filter=language_id_filter(document)):
@@ -121,6 +121,22 @@ async def _text_document_symbol(
                         self._logger.warning(
                             "Result contains DocumentSymbol and SymbolInformation results, result is skipped."
                         )
+        if document_symbols:
+
+            def traverse(symbol: DocumentSymbol, doc: TextDocument) -> None:
+                symbol.range = doc.range_to_utf16(symbol.range)
+                symbol.selection_range = doc.range_to_utf16(symbol.selection_range)
+                for child in symbol.children or []:
+                    traverse(child, doc)
+
+            for symbol in document_symbols:
+                traverse(symbol, document)
+
+        if symbol_informations:
+            for symbol_information in symbol_informations:
+                doc = await self.parent.documents.get(symbol_information.location.uri)
+                if doc is not None:
+                    symbol_information.location.range = doc.range_to_utf16(symbol_information.location.range)
 
         if document_symbols and symbol_informations:
             self._logger.warning(

packages/language_server/src/robotcode/language_server/common/parts/folding_range.py

@@ -11,6 +11,7 @@
 from robotcode.language_server.common.lsp_types import (
     FoldingRange,
     FoldingRangeParams,
+    Position,
     ServerCapabilities,
     TextDocumentIdentifier,
 )
@@ -53,6 +54,17 @@ async def _text_document_folding_range(
                 if result is not None:
                     results += result
 
-        if len(results) == 0:
+        if not results:
             return None
+
+        for result in results:
+            if result.start_character is not None:
+                result.start_character = document.position_to_utf16(
+                    Position(result.start_line, result.start_character)
+                ).character
+            if result.end_character is not None:
+                result.end_character = document.position_to_utf16(
+                    Position(result.end_line, result.end_character)
+                ).character
+
         return results

packages/language_server/src/robotcode/language_server/common/parts/hover.py

@@ -59,7 +59,7 @@ async def _text_document_hover(
         for result in await self.collect(
             self,
             document,
-            position,
+            document.position_from_utf16(position),
             callback_filter=language_id_filter(document),
         ):
             if isinstance(result, BaseException):
@@ -69,8 +69,12 @@ async def _text_document_hover(
                 if result is not None:
                     results.append(result)
 
+        for result in results:
+            if result.range is not None:
+                result.range = document.range_to_utf16(result.range)
+
         if len(results) > 0 and results[-1].contents:
-            # TODO: can we combine hover results?
+            # TODO: how can we combine hover results?
 
             return results[-1]