Improve python tracing process (#2662)

Author: tamarinvs19

utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt

@@ -10,12 +10,14 @@ import org.parsers.python.PythonParser
 import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour
 import org.utbot.framework.codegen.domain.TestFramework
 import org.utbot.framework.plugin.api.UtExecutionSuccess
+import org.utbot.python.coverage.CoverageOutputFormat
 import org.utbot.python.PythonMethodHeader
 import org.utbot.python.PythonTestGenerationConfig
 import org.utbot.python.PythonTestSet
-import org.utbot.python.utils.RequirementsInstaller
 import org.utbot.python.TestFileInformation
+import org.utbot.python.utils.RequirementsInstaller
 import org.utbot.python.code.PythonCode
+import org.utbot.python.coverage.PythonCoverageMode
 import org.utbot.python.framework.api.python.PythonClassId
 import org.utbot.python.framework.codegen.model.Pytest
 import org.utbot.python.framework.codegen.model.Unittest
@@ -114,9 +116,20 @@ class PythonGenerateTestsCommand : CliktCommand(
         .choice("PASS", "FAIL")
         .default("FAIL")
 
-    private val doNotGenerateRegressionSuite by option("--do-not-generate-regression-suite", help = "Do not generate regression test suite")
+    private val doNotGenerateRegressionSuite by option("--do-not-generate-regression-suite", help = "Do not generate regression test suite.")
+        .flag(default = false)
+
+    private val coverageMeasureMode by option("--coverage-measure-mode", help = "Use LINES or INSTRUCTIONS for coverage measurement.")
+        .choice("INSTRUCTIONS", "LINES")
+        .default("INSTRUCTIONS")
+
+    private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "Do not send coverage during execution.")
         .flag(default = false)
 
+    private val coverageOutputFormat by option("--coverage-output-format", help = "Use LINES, INSTRUCTIONS (only from function frame).")
+        .choice("INSTRUCTIONS", "LINES")
+        .default("LINES")
+
     private val testFramework: TestFramework
         get() =
             when (testFrameworkAsString) {
@@ -252,7 +265,10 @@ class PythonGenerateTestsCommand : CliktCommand(
             testSourceRootPath = Paths.get(output.toAbsolutePath()).parent.toAbsolutePath(),
             withMinimization = !doNotMinimize,
             isCanceled = { false },
-            runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour)
+            runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour),
+            coverageMeasureMode = PythonCoverageMode.parse(coverageMeasureMode),
+            sendCoverageContinuously = !doNotSendCoverageContinuously,
+            coverageOutputFormat = CoverageOutputFormat.parse(coverageOutputFormat),
         )
 
         val processor = PythonCliProcessor(

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt

@@ -10,7 +10,7 @@ package org.utbot.framework.plugin.api
  *
  * @see <a href="CONFLUENCE:Test+Minimization">Test minimization</a>
  */
-data class Instruction(
+open class Instruction(
     val internalName: String,
     val methodSignature: String,
     val lineNumber: Int,

utbot-python-executor/src/main/python/utbot_executor/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "utbot-executor"
-version = "1.7.0"
+version = "1.8.0"
 description = ""
 authors = ["Vyacheslav Tamarin <vyacheslav.tamarin@yandex.ru>"]
 readme = "README.md"

utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py

@@ -2,35 +2,49 @@
 import logging
 
 from utbot_executor.listener import PythonExecuteServer
+from utbot_executor.utils import TraceMode
 
 
-def main(hostname: str, port: int, coverage_hostname: str, coverage_port: str):
-    server = PythonExecuteServer(hostname, port, coverage_hostname, coverage_port)
+def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode, send_coverage: bool):
+    server = PythonExecuteServer(hostname, port, coverage_hostname, coverage_port, trace_mode, send_coverage)
     server.run()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        prog='UtBot Python Executor',
-        description='Listen socket stream and execute function value',
-        )
-    parser.add_argument('hostname')
-    parser.add_argument('port', type=int)
-    parser.add_argument('--logfile', default=None)
+        prog="UtBot Python Executor",
+        description="Listen socket stream and execute function value",
+    )
+    parser.add_argument("hostname")
+    parser.add_argument("port", type=int)
+    parser.add_argument("--logfile", default=None)
     parser.add_argument(
-            '--loglevel',
-            choices=["DEBUG", "INFO", "ERROR"],
-            default="ERROR",
-            )
-    parser.add_argument('coverage_hostname')
-    parser.add_argument('coverage_port', type=int)
+        "--loglevel",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+        default="ERROR",
+    )
+    parser.add_argument("coverage_hostname")
+    parser.add_argument("coverage_port", type=int)
+    parser.add_argument(
+        "--coverage_type", choices=["lines", "instructions"], default="instructions"
+    )
+    parser.add_argument(
+        "--send_coverage", action=argparse.BooleanOptionalAction
+    )
     args = parser.parse_args()
 
-    loglevel = {"DEBUG": logging.DEBUG, "INFO": logging.INFO, "ERROR": logging.ERROR}[args.loglevel]
+    loglevel = {
+        "DEBUG": logging.DEBUG,
+        "INFO": logging.INFO,
+        "WARNING": logging.WARNING,
+        "ERROR": logging.ERROR,
+    }[args.loglevel]
     logging.basicConfig(
-            filename=args.logfile,
-            format='%(asctime)s | %(levelname)s | %(funcName)s - %(message)s',
-            datefmt='%m/%d/%Y %H:%M:%S',
-            level=loglevel,
-            )
-    main(args.hostname, args.port, args.coverage_hostname, args.coverage_port)
+        filename=args.logfile,
+        format="%(asctime)s | %(levelname)s | %(funcName)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=loglevel,
+    )
+    trace_mode = TraceMode.Lines if args.coverage_type == "lines" else TraceMode.Instructions
+    send_coverage = args.send_coverage
+    main(args.hostname, args.port, args.coverage_hostname, args.coverage_port, trace_mode, send_coverage)

utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py

@@ -4,10 +4,9 @@
 import inspect
 import logging
 import pathlib
-import socket
 import sys
 import traceback
-import typing
+import types
 from typing import Any, Callable, Dict, Iterable, List, Tuple
 
 from utbot_executor.deep_serialization.deep_serialization import serialize_memory_dump, \
@@ -17,8 +16,13 @@
 from utbot_executor.deep_serialization.utils import PythonId, getattr_by_path
 from utbot_executor.memory_compressor import compress_memory
 from utbot_executor.parser import ExecutionRequest, ExecutionResponse, ExecutionFailResponse, ExecutionSuccessResponse
-from utbot_executor.ut_tracer import UtTracer
-from utbot_executor.utils import suppress_stdout as __suppress_stdout
+from utbot_executor.ut_tracer import UtTracer, UtCoverageSender
+from utbot_executor.utils import (
+    suppress_stdout as __suppress_stdout,
+    get_instructions,
+    filter_instructions,
+    TraceMode, UtInstruction,
+)
 
 __all__ = ['PythonExecutor']
 
@@ -41,9 +45,11 @@ def _load_objects(objs: List[Any]) -> MemoryDump:
 
 
 class PythonExecutor:
-    def __init__(self, coverage_hostname: str, coverage_port: int):
+    def __init__(self, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode, send_coverage: bool):
         self.coverage_hostname = coverage_hostname
         self.coverage_port = coverage_port
+        self.trace_mode = trace_mode
+        self.send_coverage = send_coverage
 
     @staticmethod
     def add_syspaths(syspaths: Iterable[str]):
@@ -91,7 +97,7 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse:
                     importlib.import_module(request.function_module),
                     request.function_name
                     )
-            if not callable(function):
+            if not isinstance(function, types.FunctionType):
                 return ExecutionFailResponse(
                         "fail",
                         f"Invalid function path {request.function_module}.{request.function_name}"
@@ -111,23 +117,26 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse:
             state_init = _update_states(loader.reload_id(), state_init_memory)
             serialized_state_init = serialize_memory_dump(state_init)
 
-            def _coverage_sender(info: typing.Tuple[str, int]):
-                if pathlib.Path(info[0]) == pathlib.Path(request.filepath):
-                    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-                    logging.debug("Coverage message: %s:%d", request.coverage_id, info[1])
-                    logging.debug("Port: %d", self.coverage_port)
-                    message = bytes(f'{request.coverage_id}:{info[1]}', encoding='utf-8')
-                    sock.sendto(message, (self.coverage_hostname, self.coverage_port))
-                    logging.debug("ID: %s, Coverage: %s", request.coverage_id, info)
+            _coverage_sender = UtCoverageSender(
+                request.coverage_id,
+                self.coverage_hostname,
+                self.coverage_port,
+                send_coverage=self.send_coverage,
+            )
 
             value = _run_calculate_function_value(
-                    function,
-                    args,
-                    kwargs,
-                    request.filepath,
-                    serialized_state_init,
-                    tracer=UtTracer(_coverage_sender)
-                    )
+                function,
+                args,
+                kwargs,
+                request.filepath,
+                serialized_state_init,
+                tracer=UtTracer(
+                    pathlib.Path(request.filepath),
+                    [sys.prefix, sys.exec_prefix],
+                    _coverage_sender,
+                    self.trace_mode,
+                ),
+            )
         except Exception as _:
             logging.debug("Error \n%s", traceback.format_exc())
             return ExecutionFailResponse("fail", traceback.format_exc())
@@ -157,7 +166,7 @@ def _serialize_state(
 
 
 def _run_calculate_function_value(
-        function: Callable,
+        function: types.FunctionType,
         args: List[Any],
         kwargs: Dict[str, Any],
         fullpath: str,
@@ -172,10 +181,8 @@ def _run_calculate_function_value(
 
     __is_exception = False
 
-    (__sources, __start, ) = inspect.getsourcelines(function)
-    __not_empty_lines = [i for i, line in enumerate(__sources, __start) if len(line.strip()) != 0]
-    logging.debug("Not empty lines %s", __not_empty_lines)
-    __end = __start + len(__sources)
+    _, __start = inspect.getsourcelines(function)
+    __all_code_stmts = filter_instructions(get_instructions(function.__code__), tracer.mode)
 
     __tracer = tracer
 
@@ -189,24 +196,23 @@ def _run_calculate_function_value(
 
     logging.debug("Coverage: %s", __tracer.counts)
     logging.debug("Fullpath: %s", fullpath)
-    module_path = pathlib.Path(fullpath)
-    __stmts = [x[1] for x in __tracer.counts if pathlib.Path(x[0]) == module_path]
-    __stmts_filtered = [x for x in __not_empty_lines if x in __stmts]
-    __stmts_filtered_with_def = [__start] + __stmts_filtered
-    __missed_filtered = [x for x in __not_empty_lines if x not in __stmts_filtered_with_def]
-    logging.debug("Covered lines: %s", __stmts_filtered_with_def)
+    __stmts_with_def = [UtInstruction(__start, 0, True)] + list(__tracer.counts.keys())
+    __missed_filtered = [x for x in __all_code_stmts if x not in __stmts_with_def]
+    logging.debug("Covered lines: %s", __stmts_with_def)
     logging.debug("Missed lines: %s", __missed_filtered)
 
+    __str_statements = [x.serialize() for x in __stmts_with_def]
+    __str_missed_statements = [x.serialize() for x in __missed_filtered]
+
     args_ids, kwargs_ids, result_id, state_after, serialized_state_after = _serialize_state(args, kwargs, __result)
     ids = args_ids + list(kwargs_ids.values())
-    # state_before, state_after = compress_memory(ids, state_before, state_after)
     diff_ids = compress_memory(ids, state_before, state_after)
 
     return ExecutionSuccessResponse(
             status="success",
             is_exception=__is_exception,
-            statements=__stmts_filtered_with_def,
-            missed_statements=__missed_filtered,
+            statements=__str_statements,
+            missed_statements=__str_missed_statements,
             state_init=state_init,
             state_before=serialized_state_before,
             state_after=serialized_state_after,

utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py

@@ -6,7 +6,7 @@
 from utbot_executor.deep_serialization.memory_objects import PythonSerializer
 from utbot_executor.parser import parse_request, serialize_response, ExecutionFailResponse
 from utbot_executor.executor import PythonExecutor
-
+from utbot_executor.utils import TraceMode
 
 RECV_SIZE = 2**15
 
@@ -17,12 +17,14 @@ def __init__(
             hostname: str,
             port: int,
             coverage_hostname: str,
-            coverage_port: str,
+            coverage_port: int,
+            trace_mode: TraceMode,
+            send_coverage: bool
             ):
         logging.info('PythonExecutor is creating...')
         self.clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.clientsocket.connect((hostname, port))
-        self.executor = PythonExecutor(coverage_hostname, coverage_port)
+        self.executor = PythonExecutor(coverage_hostname, coverage_port, trace_mode, send_coverage)
 
     def run(self) -> None:
         logging.info('PythonExecutor is ready...')
@@ -80,9 +82,9 @@ def handler(self) -> None:
                 response_size = str(len(bytes_data))
                 self.clientsocket.send((response_size + os.linesep).encode())
 
-                sended_size = 0
-                while len(bytes_data) > sended_size:
-                    sended_size += self.clientsocket.send(bytes_data[sended_size:])
+                sent_size = 0
+                while len(bytes_data) > sent_size:
+                    sent_size += self.clientsocket.send(bytes_data[sent_size:])
 
                 logging.debug('Sent all data')
         logging.info('All done...')

utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py

@@ -1,6 +1,6 @@
 import dataclasses
 import json
-from typing import Dict, List, Union
+from typing import Dict, List, Union, Tuple
 
 
 @dataclasses.dataclass
@@ -24,8 +24,8 @@ class ExecutionResponse:
 class ExecutionSuccessResponse(ExecutionResponse):
     status: str
     is_exception: bool
-    statements: List[int]
-    missed_statements: List[int]
+    statements: List[str]
+    missed_statements: List[str]
     state_init: str
     state_before: str
     state_after: str