Add gnatcoverage instrumentation support

This commit removes the gcov testsuite support, and updates the
gnatcoverage support.

Makefile now supports code coverage build. It is separated in three
steps:

(1) Generate the gnatcov runtime sources
(2) Instrument the gnatcoll-core code
(3) Compile the instrumented gnatcoll-core code

In case of gnatcoll-core out-of-tree build, GNATCOLL_OBJECT_ROOT allows
an external project to specify the gnatcoll-core root object directory.
It is useful for example for out-of-tree instrumented builds, where the
instrumented sources have previously been generated.

--gnatcov option is supported by run-tests, if GPR_PROJECT_PATH is
correctly set, which means having gnatcoll-core and gnatcov_rts
dependency. --recompile and --gnatcov can not be used simultaneously.

Co-author: Jerome Lambourg <lambourg@adacore.com>
Change-Id: Icad570f8b72b8b2d4a7c2de3306b4dcffa4fc638

Author: Jicquel

.gitignore

@@ -26,6 +26,7 @@ lib/
 *.deps
 *.pyc
 /gnat_src
+*~
 
 testsuite/debug
 testsuite/gcov

Makefile

@@ -55,6 +55,12 @@ WHICH := which
 
 # check for out-of-tree build
 SOURCE_DIR := $(dir $(MAKEFILE_LIST))
+
+# make -f with absolute path to current directory Makefile is in-tree build
+ifeq ($(SOURCE_DIR), $(shell pwd)/)
+	SOURCE_DIR := ./
+endif
+
 ifeq ($(SOURCE_DIR),./)
   RBD=
   GNATCOLL_GPR=gnatcoll.gpr
@@ -83,6 +89,7 @@ PROCESSORS    = 0
 BUILD_DIR     =
 ENABLE_SHARED = yes
 INTEGRATED    = no
+GNATCOV       =
 
 all: build
 
@@ -97,11 +104,23 @@ else
    LIBRARY_TYPES=static
 endif
 
+ifeq ($(GNATCOV), yes)
+   LIBRARY_TYPES=static
+   GNATCOV_RTS=gnatcovrts
+   GNATCOV_BUILD_OPTS=--src-subdirs=gnatcov-instr --implicit-with=gnatcov_rts
+   GNATCOV_PROJECT_PATH=GPR_PROJECT_PATH=$(CURDIR)/gnatcov_rts/share/gpr:$(GPR_PROJECT_PATH)
+else
+   GNATCOV_RTS=
+   GNATCOV_BUILD_OPTS=
+   GNATCOV_PROJECT_PATH=
+endif
+
 ifeq ($(INTEGRATED), yes)
    integrated_install=/$(NORMALIZED_TARGET)
 endif
 
 
+
 GPR_VARS=-XGNATCOLL_MMAP=$(GNATCOLL_MMAP) \
 	 -XGNATCOLL_MADVISE=$(GNATCOLL_MADVISE) \
 	 -XGNATCOLL_VERSION=$(GNATCOLL_VERSION) \
@@ -111,8 +130,9 @@ GPR_VARS=-XGNATCOLL_MMAP=$(GNATCOLL_MMAP) \
 # Used to pass extra options to GPRBUILD, like -d for instance
 GPRBUILD_OPTIONS=
 
-BUILDER=gprbuild -p -m $(GTARGET) $(RBD) -j$(PROCESSORS) $(GPR_VARS) \
-	$(GPRBUILD_OPTIONS)
+BUILDER=$(GNATCOV_PROJECT_PATH) gprbuild -p -m $(GTARGET) $(RBD) \
+        -j$(PROCESSORS) $(GPR_VARS) \
+	$(GPRBUILD_OPTIONS) $(GNATCOV_BUILD_OPTS)
 INSTALLER=gprinstall -p -f $(GTARGET) $(GPR_VARS) \
 	$(RBD) --sources-subdir=include/gnatcoll --prefix=$(prefix)$(integrated_install)
 CLEANER=gprclean -q $(RBD) $(GTARGET)
@@ -124,9 +144,20 @@ UNINSTALLER=$(INSTALLER) -p -f --install-name=gnatcoll --uninstall
 
 build: $(LIBRARY_TYPES:%=build-%)
 
-build-%:
+build-%: $(GNATCOV_RTS)
+ifeq ($(GNATCOV), yes)
+	gnatcov instrument -P $(GNATCOLL_GPR) $(RBD) \
+		--no-subprojects --level=stmt+decision
+endif
 	$(BUILDER) -XLIBRARY_TYPE=$* -XXMLADA_BUILD=$* -XGPR_BUILD=$* \
-		$(GPR_VARS) $(GNATCOLL_GPR)
+		$(GPR_VARS) $(GNATCOLL_GPR) -v
+
+###################
+# Instrumentation #
+###################
+
+gnatcovrts:
+	gnatcov setup --prefix=gnatcov_rts
 
 ###########
 # Install #

gnatcoll.gpr

@@ -55,23 +55,31 @@ project GnatColl is
 
    for Library_Kind use Library_Type;
 
+   Read_Only := external ("EXTERNALLY_BUILT", "false");
+   for Externally_Built use Read_Only;
+
+   --  If gnatcoll-core is built out of tree, and is used by an external
+   --  afterwards, then GNATCOLL_OBJECT_ROOT can be used so gnatcoll.gpr
+   --  Object_Dir and Library_Dir point to the correct directory.
+   Object_Root := external ("GNATCOLL_OBJECT_ROOT", "");
+
    case OS is
       when "windows" | "osx" =>
          --  On MacOS and Windows all object are relocatable by default
          --  thus the same object directory can be used
-         for Object_Dir use "obj/gnatcoll/all";
+         for Object_Dir use Object_Root & "obj/gnatcoll/all";
       when "unix" =>
          --  On Unix static-pic and relocatable shared the same objects
          case Library_Type is
             when "relocatable" | "static-pic" =>
-               for Object_Dir use "obj/gnatcoll/pic";
+               for Object_Dir use Object_Root & "obj/gnatcoll/pic";
             when "static" =>
-               for Object_Dir use "obj/gnatcoll/static";
+               for Object_Dir use Object_Root & "obj/gnatcoll/static";
          end case;
    end case;
 
    for Library_Dir
-      use "lib/gnatcoll/" & Project'Library_Kind;
+      use Object_Root & "lib/gnatcoll/" & Project'Library_Kind;
    for Library_Name    use Name;
 
    for Languages use ("Ada", "C");

testsuite/drivers/__init__.py

@@ -12,7 +12,6 @@
 from e3.testsuite.process import check_call
 from e3.testsuite.result import Log, TestStatus
 
-
 # Root directory of respectively the testsuite and the gnatcoll
 # repository.
 TESTSUITE_ROOT_DIR = os.path.dirname(
@@ -22,35 +21,20 @@
 DEFAULT_TIMEOUT = 5 * 60  # 5 minutes
 
 
-def make_gnatcoll(work_dir, debug=False, gcov=False, gnatcov=False):
-    """Build gnatcoll core with or without gcov instrumentation.
+def make_gnatcoll(work_dir, debug=False):
+    """Build gnatcoll core. 
 
     :param str work_dir: Working directory. GNATcoll is built in `build` subdir
         and installed in `install` subdir.
 
     :param bool debug: Whether to build GNATCOLL in debug mode. Otherwise, use
-        the prod mode. Note that gcov and gnatcov modes automatically enable
-        debug mode.
-
-    :param bool gcov: If true, build GNATCOLL with gcov instrumentation in
-        debgu mode.
-
-    :param bool gnatcov: If True, build GNATCOLL with the compile options that
-        GNATcoverage require in debug mode.
+        the prod mode.
 
     :return: A triplet (project path, source path, object path).
     :rtype: (str, str, str)
     :raise AssertError: In case compilation of installation fails.
     """
-    assert not (gcov and gnatcov)
-
-    if gcov:
-        tag = ' (gcov)'
-    elif gnatcov:
-        tag = ' (gnatcov)'
-    else:
-        tag = ''
-    logging.info('Compiling gnatcoll{}'.format(tag))
+    logging.info('Compiling gnatcoll')
 
     # Create build tree structure
     build_dir = os.path.join(work_dir, 'build')
@@ -62,16 +46,7 @@ def make_gnatcoll(work_dir, debug=False, gcov=False, gnatcov=False):
     make_gnatcoll_cmd = [
         'make', '-f', os.path.join(GNATCOLL_ROOT_DIR, 'Makefile'),
         'ENABLE_SHARED=no',
-        'BUILD={}'.format('DEBUG' if debug or gcov or gnatcov else 'PROD')]
-    if gcov:
-        make_gnatcoll_cmd += [
-            'GPRBUILD_OPTIONS=-cargs -fprofile-arcs -ftest-coverage'
-            ' -cargs:Ada -gnatwn'
-            ' -gargs']
-    elif gnatcov:
-        make_gnatcoll_cmd += [
-            'GPRBUILD_OPTIONS=-cargs -fdump-scos -fpreserve-control-flow'
-            ' -gargs']
+        'BUILD={}'.format('DEBUG' if debug else 'PROD')]
 
     # Build & Install
     p = Run(make_gnatcoll_cmd, cwd=build_dir, timeout=DEFAULT_TIMEOUT)
@@ -89,7 +64,6 @@ def make_gnatcoll(work_dir, debug=False, gcov=False, gnatcov=False):
 def gprbuild(driver,
              project_file=None,
              cwd=None,
-             gcov=False,
              scenario=None,
              gpr_project_path=None,
              timeout=DEFAULT_TIMEOUT,
@@ -103,8 +77,6 @@ def gprbuild(driver,
     :param cwd: directory in which to run gprbuild. If None the gprbuild build
         is run in the default working dir for the test.
     :type cwd: str | None
-    :param gcov: if True link with gcov libraries
-    :type gcov: bool
     :param scenario: scenario variable values
     :type scenario: dict
     :param gpr_project_path: if not None prepent this value to GPR_PROJECT_PATH
@@ -125,24 +97,13 @@ def gprbuild(driver,
                                         'support', 'test.gpr')
             scenario['TEST_SOURCES'] = driver.test_env['test_dir']
 
+    scenario_cmd = []
+    for k, v in scenario.items():
+        scenario_cmd.append('-X%s=%s' % (k, v))
+
     if cwd is None:
         cwd = driver.test_env['working_dir']
     mkdir(cwd)
-    gprbuild_cmd = [
-        'gprbuild', '--relocate-build-tree', '-p', '-P', project_file]
-    for k, v in scenario.items():
-        gprbuild_cmd.append('-X%s=%s' % (k, v))
-    if gcov:
-        gprbuild_cmd += ['-largs', '-lgcov', '-cargs',
-                         '-fprofile-arcs', '-ftest-coverage', '-g']
-    elif driver.env.gnatcov:
-        # TODO: GNATcoverage relies on debug info to do its magic. It needs
-        # consistent paths to source files in the debug info, so do not build
-        # tests with debug info, as they will reference installed sources
-        # (while GNATCOLL objects reference original sources).
-        gprbuild_cmd += ['-g0']
-    if driver.env.is_cross:
-        gprbuild_cmd.append("--target={target}".format(target=driver.env.target.triplet))
 
     # Adjust process environment
     env = kwargs.pop('env', None)
@@ -156,6 +117,36 @@ def gprbuild(driver,
             new_gpr_path += os.path.pathsep + os.environ['GPR_PROJECT_PATH']
         env['GPR_PROJECT_PATH'] = new_gpr_path
 
+    # Determine the gprbuild command line
+    gprbuild_cmd = [
+        'gprbuild', '--relocate-build-tree', '-p', '-P', project_file
+    ] + scenario_cmd
+
+    if driver.env.gnatcov:
+        from drivers.gnatcov import COVERAGE_LEVEL
+
+        gnatcov_cmd = [
+            "gnatcov", "instrument", "--level", COVERAGE_LEVEL,
+            "--relocate-build-tree",
+            "--dump-trigger=atexit",
+            "--projects", "gnatcoll",
+            "--externally-built-projects",
+            "-XEXTERNALLY_BUILT=true",
+            "--no-subprojects",
+            "-P", project_file] + scenario_cmd
+
+        check_call(driver, gnatcov_cmd,
+                   cwd=cwd,
+                   env=env,
+                   ignore_environ=ignore_environ,
+                   timeout=timeout,
+                   **kwargs)
+        gprbuild_cmd += [
+            "-margs", "-v", "-g", "-O0", "--src-subdirs=gnatcov-instr", "--implicit-with=gnatcov_rts", "-XLIBRARY_TYPE=static", "-XEXTERNALLY_BUILT=true"]
+
+    if driver.env.is_cross:
+        gprbuild_cmd.append("--target={target}".format(target=driver.env.target.triplet))
+
     check_call(
         driver,
         gprbuild_cmd,
@@ -169,8 +160,7 @@ def gprbuild(driver,
 
 
 def bin_check_call(driver, cmd, slot, test_name=None, result=None, timeout=None,
-               env=None, cwd=None):
-
+                   env=None, cwd=None, **kwargs):
     if cwd is None and "working_dir" in driver.test_env:
         cwd = driver.test_env["working_dir"]
     if result is None:
@@ -220,6 +210,7 @@ def bin_check_call(driver, cmd, slot, test_name=None, result=None, timeout=None,
 
     # Append the status code and process output to the log to ease post-mortem
     # investigation.
+    result.log += f"Cmd_Line: {' '.join(cmd)}"
     result.log += "Status code: {}\n".format(process.status)
     result.log += "Output:\n"
 
@@ -245,13 +236,13 @@ def run_test_program(driver, cmd, slot, test_name=None, result=None, **kwargs):
     Run a test program. This dispatches to running it under Valgrind or
     "gnatcov run", depending on the testsuite options.
     """
-    from drivers.gnatcov import gnatcov_run
+    from drivers.gnatcov import gnatcov_call
     from drivers.valgrind import check_call_valgrind
 
     if driver.env.valgrind:
         wrapper = check_call_valgrind
     elif driver.env.gnatcov:
-        wrapper = gnatcov_run
+        wrapper = gnatcov_call
     else:
         wrapper = bin_check_call
 

testsuite/drivers/basic.py

@@ -49,7 +49,7 @@ def run(self):
             gpr_project_path = self.env.gnatcoll_debug_gpr_dir
         else:
             gpr_project_path = self.env.gnatcoll_gpr_dir
-        gprbuild(self, gcov=self.env.gcov, gpr_project_path=gpr_project_path)
+        gprbuild(self, gpr_project_path=gpr_project_path)
 
         # Copy the requested data files
         for data in self.test_env.get('data', []):

testsuite/drivers/build_and_run.py

@@ -29,7 +29,6 @@ def run(self):
         gprbuild(
             self,
             project_file="test.gpr",
-            gcov=self.env.gcov,
             gpr_project_path=gpr_project_path
         )
 

testsuite/drivers/build_run_diff.py

@@ -68,7 +68,6 @@ def run(self):
         gprbuild(
             self,
             project_file="test.gpr",
-            gcov=self.env.gcov,
             gpr_project_path=gpr_project_path
         )
 
@@ -81,7 +80,14 @@ def run(self):
                 timeout=self.default_process_timeout
             )
         else:
-            p = self.shell(["bash", "test.sh"], catch_error=False)
+            p = run_test_program(
+                self,
+                ["bash", self.working_dir("test.sh")],
+                self.slot,
+                timeout=self.default_process_timeout,
+            )
+            # Output explicitly to be compared with the expected output.
+            self.output += p.out.decode('utf-8')
 
         if p.status:
             self.output += ">>>program returned status code {}\n".format(

testsuite/drivers/data_validation.py

@@ -82,5 +82,4 @@ def tear_down(self, previous_values, slot):
             rm(self.test_env['working_dir'], recursive=True)
 
     def build(self, previous_values, slot):
-        return gprbuild(self, gcov=self.env.gcov,
-                        gpr_project_path=self.env.gnatcoll_gpr_dir)
+        return gprbuild(self, gpr_project_path=self.env.gnatcoll_gpr_dir)