feat(analyze): add CLI options for pythonpath, variable, and variablefile to analyze code command; collect errors for unimportable command line variable files

Author: d-biehl

.vscode/launch.json

@@ -66,7 +66,11 @@
 				// "E:\\source\\uvtestprj\\tests\\first.robotrepl"
 				"analyze",
 				"code",
-				"tests"
+				// "tests"
+				// "repl",
+				// "-v",
+				// "asd:asd"
+
 			]
 		},
 		{

packages/analyze/src/robotcode/analyze/cli.py

@@ -1,20 +1,23 @@
 from pathlib import Path
 from textwrap import indent
-from typing import Set, Tuple
+from typing import List, Optional, Set, Tuple
 
 import click
 
 from robotcode.analyze.config import AnalyzeConfig
-from robotcode.core.lsp.types import DiagnosticSeverity
+from robotcode.core.lsp.types import Diagnostic, DiagnosticSeverity
 from robotcode.core.text_document import TextDocument
+from robotcode.core.uri import Uri
+from robotcode.core.utils.path import try_get_relative_path
+from robotcode.core.workspace import WorkspaceFolder
 from robotcode.plugin import Application, pass_application
 from robotcode.robot.config.loader import (
     load_robot_config_from_path,
 )
 from robotcode.robot.config.utils import get_config_files
 
 from .__version__ import __version__
-from .code_analyzer import CodeAnalyzer
+from .code_analyzer import CodeAnalyzer, DocumentDiagnosticReport, FolderDiagnosticReport
 
 
 @click.group(
@@ -44,6 +47,7 @@ def analyze(app: Application) -> None:
 
 class Statistic:
     def __init__(self) -> None:
+        self.folders: Set[WorkspaceFolder] = set()
         self.files: Set[TextDocument] = set()
         self.errors = 0
         self.warnings = 0
@@ -69,6 +73,7 @@ def __str__(self) -> str:
     "-f",
     "--filter",
     "filter",
+    metavar="PATTERN",
     type=str,
     multiple=True,
     help="""\
@@ -86,25 +91,32 @@ def __str__(self) -> str:
 @click.option(
     "-V",
     "--variablefile",
-    metavar="path",
+    metavar="PATH",
     type=str,
     multiple=True,
     help="Python or YAML file file to read variables from. see `robot --variablefile` option.",
 )
 @click.option(
     "-P",
     "--pythonpath",
-    metavar="path",
+    metavar="PATH",
     type=str,
     multiple=True,
-    help="Additional locations (directories, ZIPs, JARs) where to search test libraries"
+    help="Additional locations where to search test libraries"
     " and other extensions when they are imported. see `robot --pythonpath` option.",
 )
 @click.argument(
     "paths", nargs=-1, type=click.Path(exists=True, dir_okay=True, file_okay=True, readable=True, path_type=Path)
 )
 @pass_application
-def code(app: Application, filter: Tuple[str], paths: Tuple[Path]) -> None:
+def code(
+    app: Application,
+    filter: Tuple[str],
+    variable: Tuple[str, ...],
+    variablefile: Tuple[str, ...],
+    pythonpath: Tuple[str, ...],
+    paths: Tuple[Path],
+) -> None:
     """\
         Performs static code analysis to detect syntax errors, missing keywords or variables,
         missing arguments, and more on the given *PATHS*. *PATHS* can be files or directories.
@@ -132,35 +144,45 @@ def code(app: Application, filter: Tuple[str], paths: Tuple[Path]) -> None:
             *(app.config.profiles or []), verbose_callback=app.verbose, error_callback=app.error
         ).evaluated_with_env()
 
+        if variable:
+            if robot_profile.variables is None:
+                robot_profile.variables = {}
+            for v in variable:
+                name, value = v.split(":", 1) if ":" in v else (v, "")
+                robot_profile.variables.update({name: value})
+
+        if pythonpath:
+            if robot_profile.python_path is None:
+                robot_profile.python_path = []
+            robot_profile.python_path.extend(pythonpath)
+
+        if variablefile:
+            if robot_profile.variable_files is None:
+                robot_profile.variable_files = []
+            for vf in variablefile:
+                robot_profile.variable_files.append(vf)
+
         statistics = Statistic()
         for e in CodeAnalyzer(
             app=app,
             analysis_config=analyzer_config.to_workspace_analysis_config(),
             robot_profile=robot_profile,
             root_folder=root_folder,
         ).run(paths=paths, filter=filter):
-            statistics.files.add(e.document)
-
-            doc_path = e.document.uri.to_path().relative_to(root_folder) if root_folder else e.document.uri.to_path()
-            if e.items:
-
-                for item in e.items:
-                    severity = item.severity if item.severity is not None else DiagnosticSeverity.ERROR
-
-                    if severity == DiagnosticSeverity.ERROR:
-                        statistics.errors += 1
-                    elif severity == DiagnosticSeverity.WARNING:
-                        statistics.warnings += 1
-                    elif severity == DiagnosticSeverity.INFORMATION:
-                        statistics.infos += 1
-                    elif severity == DiagnosticSeverity.HINT:
-                        statistics.hints += 1
-
-                    app.echo(
-                        f"{doc_path}:{item.range.start.line + 1}:{item.range.start.character + 1}: "
-                        + click.style(f"[{severity.name[0]}] {item.code}", fg=SEVERITY_COLORS[severity])
-                        + f": {indent(item.message, prefix='  ').strip()}",
-                    )
+            if isinstance(e, FolderDiagnosticReport):
+                statistics.folders.add(e.folder)
+
+                if e.items:
+                    _print_diagnostics(app, root_folder, statistics, e.items, e.folder.uri.to_path())
+
+            elif isinstance(e, DocumentDiagnosticReport):
+                statistics.files.add(e.document)
+
+                doc_path = (
+                    e.document.uri.to_path().relative_to(root_folder) if root_folder else e.document.uri.to_path()
+                )
+                if e.items:
+                    _print_diagnostics(app, root_folder, statistics, e.items, doc_path)
 
         statistics_str = str(statistics)
         if statistics.errors > 0:
@@ -172,3 +194,51 @@ def code(app: Application, filter: Tuple[str], paths: Tuple[Path]) -> None:
 
     except (TypeError, ValueError) as e:
         raise click.ClickException(str(e)) from e
+
+
+def _print_diagnostics(
+    app: Application,
+    root_folder: Optional[Path],
+    statistics: Statistic,
+    diagnostics: List[Diagnostic],
+    folder_path: Optional[Path],
+    print_range: bool = True,
+) -> None:
+    for item in diagnostics:
+        severity = item.severity if item.severity is not None else DiagnosticSeverity.ERROR
+
+        if severity == DiagnosticSeverity.ERROR:
+            statistics.errors += 1
+        elif severity == DiagnosticSeverity.WARNING:
+            statistics.warnings += 1
+        elif severity == DiagnosticSeverity.INFORMATION:
+            statistics.infos += 1
+        elif severity == DiagnosticSeverity.HINT:
+            statistics.hints += 1
+
+        app.echo(
+            (
+                (
+                    f"{folder_path}:"
+                    + (f"{item.range.start.line + 1}:{item.range.start.character + 1}: " if print_range else " ")
+                )
+                if folder_path and folder_path != root_folder
+                else " "
+            )
+            + click.style(f"[{severity.name[0]}] {item.code}", fg=SEVERITY_COLORS[severity])
+            + f": {indent(item.message, prefix='  ').strip()}",
+        )
+
+        if item.related_information:
+            for related in item.related_information or []:
+                related_path = try_get_relative_path(Uri(related.location.uri).to_path(), root_folder)
+
+                app.echo(
+                    f"    {related_path}:"
+                    + (
+                        f"{related.location.range.start.line + 1}:{related.location.range.start.character + 1}: "
+                        if print_range
+                        else " "
+                    )
+                    + f"{indent(related.message, prefix='      ').strip()}",
+                )

packages/analyze/src/robotcode/analyze/code_analyzer.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Iterable, List, Optional
+from typing import Iterable, List, Optional, Union
 
 from robotcode.core.ignore_spec import IgnoreSpec
 from robotcode.core.lsp.types import Diagnostic
@@ -22,6 +22,12 @@ class DocumentDiagnosticReport:
     items: List[Diagnostic]
 
 
+@dataclass
+class FolderDiagnosticReport:
+    folder: WorkspaceFolder
+    items: List[Diagnostic]
+
+
 class CodeAnalyzer(DiagnosticsContext):
     def __init__(
         self,
@@ -72,25 +78,40 @@ def workspace(self) -> Workspace:
     def diagnostics(self) -> DiagnosticHandlers:
         return self._dispatcher
 
-    def run(self, paths: Iterable[Path] = {}, filter: Iterable[str] = {}) -> Iterable[DocumentDiagnosticReport]:
+    def run(
+        self, paths: Iterable[Path] = {}, filter: Iterable[str] = {}
+    ) -> Iterable[Union[DocumentDiagnosticReport, FolderDiagnosticReport]]:
         for folder in self.workspace.workspace_folders:
+            self.app.verbose(f"Initialize folder {folder.uri.to_path()}")
+            initialize_result = self.diagnostics.initialize_folder(folder)
+            if initialize_result is not None:
+                diagnostics: List[Diagnostic] = []
+                for item in initialize_result:
+                    if item is None:
+                        continue
+                    elif isinstance(item, BaseException):
+                        self.app.error(f"Error analyzing {folder.uri.to_path()}: {item}")
+                    else:
+                        diagnostics.extend(item)
+                if diagnostics:
+                    yield FolderDiagnosticReport(folder, diagnostics)
 
             documents = self.collect_documents(folder, paths=paths, filter=filter)
 
             self.app.verbose(f"Analyzing {len(documents)} documents")
             for document in documents:
                 analyze_result = self.diagnostics.analyze_document(document)
                 if analyze_result is not None:
-                    diagnostics: List[Diagnostic] = []
+                    diagnostics = []
                     for item in analyze_result:
                         if item is None:
                             continue
                         elif isinstance(item, BaseException):
-                            self.app.error(f"Error analyzing {document.uri}: {item}")
+                            self.app.error(f"Error analyzing {document.uri.to_path()}: {item}")
                         else:
                             diagnostics.extend(item)
-
-                    yield DocumentDiagnosticReport(document, diagnostics)
+                    if diagnostics:
+                        yield DocumentDiagnosticReport(document, diagnostics)
 
             self.app.verbose(f"Collect Diagnostics for {len(documents)} documents")
             for document in documents:
@@ -101,11 +122,11 @@ def run(self, paths: Iterable[Path] = {}, filter: Iterable[str] = {}) -> Iterabl
                         if item is None:
                             continue
                         elif isinstance(item, BaseException):
-                            self.app.error(f"Error analyzing {document.uri}: {item}")
+                            self.app.error(f"Error analyzing {document.uri.to_path()}: {item}")
                         else:
                             diagnostics.extend(item)
-
-                    yield DocumentDiagnosticReport(document, diagnostics)
+                    if diagnostics:
+                        yield DocumentDiagnosticReport(document, diagnostics)
 
     def collect_documents(
         self, folder: WorkspaceFolder, paths: Iterable[Path] = {}, filter: Iterable[str] = {}

packages/analyze/src/robotcode/analyze/diagnostics_context.py

@@ -5,20 +5,29 @@
 from robotcode.core.language import language_id_filter
 from robotcode.core.lsp.types import Diagnostic
 from robotcode.core.text_document import TextDocument
-from robotcode.core.workspace import Workspace
+from robotcode.core.workspace import Workspace, WorkspaceFolder
 from robotcode.robot.config.model import RobotBaseProfile
 from robotcode.robot.diagnostics.workspace_config import WorkspaceAnalysisConfig
 
 
 class DiagnosticHandlers:
     @event
-    def analyzers(sender, document: TextDocument) -> Optional[List[Diagnostic]]: ...
+    def document_analyzers(sender, document: TextDocument) -> Optional[List[Diagnostic]]: ...
+    @event
+    def folder_initializers(sender, folder: WorkspaceFolder) -> Optional[List[Diagnostic]]: ...
 
     @event
     def collectors(sender, document: TextDocument) -> Optional[List[Diagnostic]]: ...
 
+    def initialize_folder(self, folder: WorkspaceFolder) -> List[Union[List[Diagnostic], BaseException, None]]:
+        return self.folder_initializers(
+            self,
+            folder,
+            return_exceptions=True,
+        )
+
     def analyze_document(self, document: TextDocument) -> List[Union[List[Diagnostic], BaseException, None]]:
-        return self.analyzers(
+        return self.document_analyzers(
             self,
             document,
             callback_filter=language_id_filter(document),

packages/analyze/src/robotcode/analyze/robot_framework_language_provider.py

@@ -1,3 +1,4 @@
+import glob
 import sys
 from pathlib import Path
 from typing import Any, Iterable, List, Optional
@@ -46,7 +47,8 @@ def __init__(self, diagnostics_context: DiagnosticsContext) -> None:
         )
 
         self.diagnostics_context.workspace.documents.on_read_document_text.add(self.on_read_document_text)
-        self.diagnostics_context.diagnostics.analyzers.add(self.analyze_document)
+        self.diagnostics_context.diagnostics.folder_initializers.add(self.analyze_folder)
+        self.diagnostics_context.diagnostics.document_analyzers.add(self.analyze_document)
 
     def _update_python_path(self) -> None:
         if self.diagnostics_context.workspace.root_uri is not None:
@@ -56,8 +58,9 @@ def _update_python_path(self) -> None:
                     pa = Path(self.diagnostics_context.workspace.root_uri.to_path(), pa)
 
                 absolute_path = str(pa.absolute())
-                if absolute_path not in sys.path:
-                    sys.path.insert(0, absolute_path)
+                for f in glob.glob(absolute_path):
+                    if Path(f).is_dir() and f not in sys.path:
+                        sys.path.insert(0, f)
 
     @language_id("robotframework")
     def on_read_document_text(self, sender: Any, uri: Uri) -> str:
@@ -95,3 +98,8 @@ def analyze_document(self, sender: Any, document: TextDocument) -> Optional[List
         namespace.analyze()
 
         return self._document_cache.get_diagnostic_modifier(document).modify_diagnostics(namespace.get_diagnostics())
+
+    def analyze_folder(self, sender: Any, folder: WorkspaceFolder) -> Optional[List[Diagnostic]]:
+        imports_manager = self._document_cache.get_imports_manager_for_workspace_folder(folder)
+
+        return imports_manager.diagnostics

packages/core/src/robotcode/core/lsp/types.py

@@ -3988,6 +3988,18 @@ def is_in_range(self, range: Range, include_end: bool = True) -> bool:
     def __hash__(self) -> int:
         return hash((self.start, self.end))
 
+    @staticmethod
+    def from_int_range(
+        start_line: int, start_character: int = 0, end_line: Optional[int] = None, end_character: Optional[int] = None
+    ) -> Range:
+        return Range(
+            start=Position(line=start_line, character=start_character),
+            end=Position(
+                line=end_line if end_line is not None else start_line,
+                character=end_character if end_character is not None else start_character,
+            ),
+        )
+
 
 @dataclass
 class WorkspaceFoldersChangeEvent(CamelSnakeMixin):