Create a standalone tool for coverage measurement (#245)

robertsipka · rtakacs · commit 8313769f209b · 2018-12-17T13:07:09.000+01:00
JSRemoteTest-DCO-1.0-Signed-off-by: Robert Sipka rsipka.uszeged@partner.samsung.com
diff --git a/.travis.yml b/.travis.yml
@@ -22,7 +22,6 @@ env:
     - OPTS="--device=rpi2 --app=iotjs" RUN_DOCKER=yes
     - OPTS="--device=rpi2 --app=jerryscript" RUN_DOCKER=yes
     - OPTS="--device=artik530 --app=iotjs" RUN_DOCKER=yes
-    - OPTS="--device=artik530 --app=jerryscript" RUN_DOCKER=yes
     - OPTS="--device=artik053 --app=iotjs" RUN_DOCKER=yes
     - OPTS="--device=artik053 --app=jerryscript" RUN_DOCKER=yes
     - OPTS="--device=stm32f4dis --app=iotjs" RUN_DOCKER=yes
diff --git a/jstest/__main__.py b/jstest/__main__.py
@@ -84,8 +84,8 @@ def parse_options():
                         help='do not measure memory statistics (default: %(default)s)')
 
     parser.add_argument('--coverage',
-                        metavar='SERVER_ADDRESS(HOST:PORT)',
-                        help='use jerry-debugger to calculate the JS source code coverage')
+                        action='store_true', default=False,
+                        help='calculate the JS source code coverage (default: %(default)s)')
 
     parser.add_argument('--quiet',
                         action='store_true', default=False,
@@ -152,17 +152,17 @@ def adjust_options(options):
     '''
     if options.device == 'artik530' and options.app == 'jerryscript':
         jstest.console.warning('JerryScript is not supported for Tizen.')
-        options.no_build = True
-        options.no_flash = True
-        options.no_test = True
+        sys.exit(1)
 
     # TODO: resolve this section, debugger should work on every target.
     if options.debugger:
         if options.device == 'stm32f4dis':
-            jstest.console.warning('Debugger is not supported on STM32F4-Discovery')
+            jstest.console.warning('Debugger is disabled, beacuse it is not supported on'
+                                   ' STM32F4-Discovery')
             options.debugger = None
         elif options.device == 'artik053' and options.app == 'jerryscript':
-            jstest.console.warning('Debugger is not supported on ARTIK053 with JerryScript')
+            jstest.console.warning('Debugger is disabled, because it is not supported on'
+                                   ' ARTIK053 with JerryScript')
             options.debugger = None
 
     if options.emulate:
@@ -183,12 +183,18 @@ def adjust_options(options):
             atexit.register(pseudo_terminal.close_pseudo_terminal, options)
 
     if options.coverage:
+        if not options.debugger or options.debugger == 'no_address':
+            jstest.console.error('Coverage measurement is require the enabled debugger option'
+                                 ' with a valid server ADDRESS')
+            sys.exit(1)
+
         if options.app != 'iotjs':
             jstest.console.warning('Coverage measurement is only supported with IoT.js!')
-            options.coverage = None
+            sys.exit(1)
 
-        elif options.buildtype != 'debug':
-            jstest.console.warning('Coverage measurement is only supported with debug build type!')
+        if options.buildtype != 'debug':
+            jstest.console.warning('Buidltype was set to debug because the coverage measurement is'
+                                   ' only supported with debug build type!')
             # Overwrite the buildtype option to debug.
             # In IoT.js the code is minimized in release mode, which will mess up the line numbers.
             options.buildtype = 'debug'
diff --git a/jstest/resources/patches/coverage-client.diff b/jstest/resources/patches/coverage-client.diff
diff --git a/jstest/resources/patches/jerryscript-coverage.diff b/jstest/resources/patches/jerryscript-coverage.diff
@@ -1,29 +1,17 @@
-diff --git a/jerry-core/debugger/debugger-ws.c b/jerry-core/debugger/debugger-ws.c
-index 0f12a69..eac1ac2 100644
---- a/jerry-core/debugger/debugger-ws.c
-+++ b/jerry-core/debugger/debugger-ws.c
-@@ -22,6 +22,7 @@
- #include <arpa/inet.h>
- #include <errno.h>
- #include <fcntl.h>
-+#include <sys/socket.h>
- #include <unistd.h>
- 
- /* JerryScript debugger protocol is a simplified version of RFC-6455 (WebSockets). */
 diff --git a/jerry-core/parser/js/js-parser-statm.c b/jerry-core/parser/js/js-parser-statm.c
-index 55bd8b9..f73ccd5 100644
+index 4330ef4..26f8899 100644
 --- a/jerry-core/parser/js/js-parser-statm.c
 +++ b/jerry-core/parser/js/js-parser-statm.c
-@@ -341,7 +341,7 @@ parser_parse_var_statement (parser_context_t *context_p) /**< context */
-         cbc_argument_t last_cbc = context_p->last_cbc;
-         context_p->last_cbc_opcode = PARSER_CBC_UNAVAILABLE;
- 
+@@ -332,7 +332,7 @@ parser_parse_var_statement (parser_context_t *context_p) /**< context */
+       if ((JERRY_CONTEXT (debugger_flags) & JERRY_DEBUGGER_CONNECTED)
+           && ident_line_counter != context_p->last_breakpoint_line)
+       {
 -        parser_emit_cbc (context_p, CBC_BREAKPOINT_DISABLED);
 +        parser_emit_cbc (context_p, CBC_BREAKPOINT_ENABLED);
          parser_flush_cbc (context_p);
  
          parser_append_breakpoint_info (context_p, JERRY_DEBUGGER_BREAKPOINT_LIST, ident_line_counter);
-@@ -1707,7 +1707,7 @@ parser_parse_statements (parser_context_t *context_p) /**< context */
+@@ -1751,7 +1751,7 @@ parser_parse_statements (parser_context_t *context_p) /**< context */
        {
          JERRY_ASSERT (context_p->last_breakpoint_line == 0);
  
@@ -32,7 +20,7 @@ index 55bd8b9..f73ccd5 100644
          parser_flush_cbc (context_p);
  
          parser_append_breakpoint_info (context_p, JERRY_DEBUGGER_BREAKPOINT_LIST, context_p->token.line);
-@@ -1777,7 +1777,7 @@ parser_parse_statements (parser_context_t *context_p) /**< context */
+@@ -1821,7 +1821,7 @@ parser_parse_statements (parser_context_t *context_p) /**< context */
          && context_p->token.type != LEXER_KEYW_CASE
          && context_p->token.type != LEXER_KEYW_DEFAULT)
      {
@@ -42,10 +30,10 @@ index 55bd8b9..f73ccd5 100644
  
        parser_append_breakpoint_info (context_p, JERRY_DEBUGGER_BREAKPOINT_LIST, context_p->token.line);
 diff --git a/jerry-core/parser/js/js-parser.c b/jerry-core/parser/js/js-parser.c
-index bdda834..41d7bff 100644
+index 4f145e3..707ca01 100644
 --- a/jerry-core/parser/js/js-parser.c
 +++ b/jerry-core/parser/js/js-parser.c
-@@ -1433,7 +1433,7 @@ parser_post_processing (parser_context_t *context_p) /**< context */
+@@ -1492,7 +1492,7 @@ parser_post_processing (parser_context_t *context_p) /**< context */
        && !(context_p->status_flags & PARSER_DEBUGGER_BREAKPOINT_APPENDED))
    {
      /* Always provide at least one breakpoint. */
@@ -54,7 +42,7 @@ index bdda834..41d7bff 100644
      parser_flush_cbc (context_p);
  
      parser_append_breakpoint_info (context_p, JERRY_DEBUGGER_BREAKPOINT_LIST, context_p->token.line);
-@@ -1819,7 +1819,7 @@ parser_post_processing (parser_context_t *context_p) /**< context */
+@@ -1887,7 +1887,7 @@ parser_post_processing (parser_context_t *context_p) /**< context */
      flags = cbc_flags[opcode];
  
  #ifdef JERRY_DEBUGGER
diff --git a/jstest/resources/patches/tizenrt-coverage.diff b/jstest/resources/patches/tizenrt-coverage.diff
diff --git a/jstest/resources/resources.json b/jstest/resources/resources.json
@@ -80,11 +80,8 @@
             "file": "%{patches}/tizenrt-mbedtls.diff"
           },
           {
-            "condition": "'%{appname}' == 'iotjs' and %{coverage}",
-            "file": "%{patches}/tizenrt-coverage.diff"
-          },
-          {
-            "file": "%{patches}/tizenrt-openocd.diff"
+            "file": "%{patches}/tizenrt-openocd.diff",
+            "revert": false
           }
         ]
       }
@@ -191,7 +188,7 @@
         "linker-map": "%{iotjs}/build/%{target}/%{build-type}/iotjs.map",
         "libdir": "%{iotjs}/build/%{target}/%{build-type}/lib",
         "image": "%{iotjs}/build/%{target}/%{build-type}/bin/iotjs",
-        "coverage-client": "%{iotjs}/deps/jerry/jerry-debugger/jerry-client-ws.py",
+        "coverage-client": "%{js-remote-test}/tools/coverage_client.py",
         "js-sources" : "%{iotjs}/src/js/"
       },
       "config": [
@@ -230,11 +227,6 @@
             "file": "%{patches}/jerryscript-coverage.diff",
             "submodule": "%{iotjs}/deps/jerry/"
           },
-          {
-            "condition": "'%{device}' in ['artik053', 'artik530', 'rpi2'] and %{coverage}",
-            "file": "%{patches}/coverage-client.diff",
-            "submodule": "%{iotjs}/deps/jerry/"
-          },
           {
             "condition": "'%{device}' in ['artik530', 'rpi2'] and %{memstat}",
             "file": "%{patches}/linux-iotjs-stack.diff"
diff --git a/jstest/testrunner/devices/artik053.py b/jstest/testrunner/devices/artik053.py
@@ -51,7 +51,7 @@ def execute(self, testset, test):
         self.channel.putc(command[self.app])
         self.channel.readline()
 
-        if self.env.options.coverage and self.app == 'iotjs':
+        if self.env.options.coverage:
             # Start the client script on a different thread for coverage.
             client_thread = Thread(target=testrunner_utils.run_coverage_script,
                                    kwargs={'env': self.env})
diff --git a/jstest/testrunner/utils.py b/jstest/testrunner/utils.py
@@ -93,6 +93,8 @@ def parse_coverage_info(env, coverage_output):
                 continue
 
             filename, _ = os.path.splitext(js_name)
+            if filename not in coverage_info:
+                continue
 
             # Iterate reached js files.
             for line_number, line_value in raw_data[js_name].iteritems():
@@ -159,7 +161,7 @@ def run_coverage_script(env):
     # Add latency because the start up of the debug server needs time.
     time.sleep(2)
 
-    address = env.options.coverage
+    address = env.options.debugger
     iotjs = env.modules.iotjs
     coverage_client = iotjs.paths['coverage-client']
     device = env.options.device
@@ -171,9 +173,8 @@ def run_coverage_script(env):
     result_path = utils.join(result_dir, result_name)
 
     utils.mkdir(result_dir)
-    utils.execute(paths.PROJECT_ROOT, coverage_client, ['--non-interactive',
-                                                        '--coverage-output=%s' % result_path,
-                                                        address])
+    utils.execute(paths.PROJECT_ROOT, 'python',
+                  [coverage_client, '--coverage-output', result_path, address])
 
 
 def read_port_from_url(url):
diff --git a/tools/coverage_client.py b/tools/coverage_client.py
