Add comment parsing from CastXML.

In CastXML 3.6, a new feature was introduced to capture the
documentation comments added to the header files of the C++ sent through
the program.  Add the code to pygccxml to read those new comment nodes and
attributes into a new "comment" declaration type.

Author: josephsnyder

pygccxml/declarations/comment.py

@@ -0,0 +1,52 @@
+# Copyright 2014-2017 Insight Software Consortium.
+# Copyright 2004-2009 Roman Yakovenko.
+# Distributed under the Boost Software License, Version 1.0.
+# See http://www.boost.org/LICENSE_1_0.txt
+
+"""
+Describe a C++ comment declaration.
+
+"""
+
+
+from . import declaration
+
+
+class comment_t(declaration.declaration_t):
+
+    def __init__(self, name='', declarations=None):
+        """
+        Creates an object that describes a C++ comment declaration.
+
+        Args:
+
+        """
+        declaration.declaration_t.__init__(self, name)
+        self.location = {}
+        self._start_line = 0
+        self._end_line = 0
+        self._text = ""
+
+    @property
+    def start_line(self):
+        return self._start_line
+
+    @start_line.setter
+    def start_line(self, start_line):
+        self._start_line = int(start_line)
+
+    @property
+    def end_line(self):
+        return self._end_line
+
+    @end_line.setter
+    def end_line(self, end_line):
+        self._end_line = int(end_line)
+
+    @property
+    def text(self):
+        return self._text
+
+    @text.setter
+    def text(self, text):
+        self._text = text

pygccxml/declarations/decl_factory.py

@@ -12,6 +12,7 @@
 from .calldef_members import destructor_t
 from .calldef_members import member_operator_t
 from .calldef_members import casting_operator_t
+from .comment import comment_t
 from .free_calldef import free_function_t
 from .free_calldef import free_operator_t
 from .enumeration import enumeration_t
@@ -89,3 +90,7 @@ def create_typedef(self, *arguments, **keywords):
     def create_variable(self, *arguments, **keywords):
         """creates instance of class that describes variable declaration"""
         return variable_t(*arguments, **keywords)
+
+    def create_comment(self, *arguments, **keywords):
+        """creates instance of class that describes variable declaration"""
+        return comment_t(*arguments, **keywords)

pygccxml/declarations/decl_visitor.py

@@ -57,3 +57,6 @@ def visit_typedef(self):
 
     def visit_variable(self):
         raise NotImplementedError()
+
+    def visit_comment(self):
+        raise NotImplementedError()

pygccxml/declarations/declaration.py

@@ -37,6 +37,7 @@ def __init__(
         self._cache = algorithms_cache.declaration_algs_cache_t()
         self._partial_name = None
         self._decorated_name = None
+        self._comment = None
 
     def __str__(self):
         """
@@ -338,3 +339,11 @@ def _warn_deprecated():
             "Please use the declarations.get_dependencies_from_decl()" +
             "function instead.\n",
             DeprecationWarning)
+
+    @property
+    def comment(self):
+        return self._comment
+
+    @comment.setter
+    def comment(self, comment):
+        self._comment = comment

pygccxml/parser/linker.py

@@ -107,6 +107,9 @@ def visit_destructor(self):
     def visit_member_operator(self):
         self.__link_calldef()
 
+    def visit_comment(self):
+        pass
+
     def visit_casting_operator(self):
         self.__link_calldef()
         # FIXME: is the patch still needed as the demangled name support has

pygccxml/parser/scanner.py

@@ -16,15 +16,18 @@
 XML_AN_ACCESS = "access"
 XML_AN_ALIGN = "align"
 XML_AN_ARTIFICIAL = "artificial"
+XML_AN_ATTACHED = "attached"
 XML_AN_ATTRIBUTES = "attributes"
 XML_AN_BASE_TYPE = "basetype"
 XML_AN_BASES = "bases"
 XML_AN_BITS = "bits"
+XML_AN_COMMENT = "comment"
 XML_AN_CONST = "const"
 XML_AN_CONTEXT = "context"
 XML_AN_CVS_REVISION = "cvs_revision"
 XML_AN_CASTXML_FORMAT = "format"
 XML_AN_DEFAULT = "default"
+XML_AN_END_LINE = "end_line"
 XML_AN_EXPLICIT = "explicit"
 XML_AN_EXTERN = "extern"
 XML_AN_FILE = "file"
@@ -33,6 +36,7 @@
 XML_AN_INIT = "init"
 XML_AN_INLINE = "inline"
 XML_AN_LINE = "line"
+XML_AN_BEGIN_LINE = "begin_line"
 XML_AN_MANGLED = "mangled"
 XML_AN_MAX = "max"
 XML_AN_MEMBERS = "members"
@@ -52,6 +56,7 @@
 XML_NN_ARRAY_TYPE = "ArrayType"
 XML_NN_CASTING_OPERATOR = "Converter"
 XML_NN_CLASS = "Class"
+XML_NN_COMMENT = "Comment"
 XML_NN_CONSTRUCTOR = "Constructor"
 XML_NN_CV_QUALIFIED_TYPE = "CvQualifiedType"
 XML_NN_DESTRUCTOR = "Destructor"
@@ -110,6 +115,7 @@ def __init__(self, xml_file, decl_factory, config, *args):
             XML_NN_UNION: self.__read_union,
             XML_NN_FIELD: self.__read_field,
             XML_NN_CASTING_OPERATOR: self.__read_casting_operator,
+            XML_NN_COMMENT: self.__read_comment,
             XML_NN_CONSTRUCTOR: self.__read_constructor,
             XML_NN_DESTRUCTOR: self.__read_destructor,
             XML_NN_FUNCTION: self.__read_function,
@@ -125,6 +131,7 @@ def __init__(self, xml_file, decl_factory, config, *args):
             XML_NN_DESTRUCTOR,
             XML_NN_ENUMERATION,
             XML_NN_FILE,
+            XML_NN_COMMENT,
             XML_NN_FUNCTION,
             XML_NN_FREE_OPERATOR,
             XML_NN_MEMBER_OPERATOR,
@@ -186,6 +193,19 @@ def xml_generator_from_xml_file(self):
     def read(self):
         xml.sax.parse(self.xml_file, self)
 
+    def _handle_comment(self, declaration):
+        comm_decl = self.__declarations.get(declaration.comment)
+        if comm_decl:
+            with open(self.__files.get(comm_decl.location.file_name, "r")) as file:
+                line_list = file.readlines()
+                comment_text = []
+                for indx in range(comm_decl.start_line - 1,comm_decl.end_line):
+                    comm_line = line_list[indx]
+                    comm_line = comm_line.strip("\n")
+                    comment_text.append(comm_line)
+                comm_decl.text = comment_text
+        return comm_decl
+
     def endDocument(self):
         # updating membership
         members_mapping = {}
@@ -195,6 +215,8 @@ def endDocument(self):
                 continue
             members_mapping[id(decl)] = members
         self.__members = members_mapping
+        for gccxml_id, decl in self.__declarations.items():
+            decl.comment = self._handle_comment(decl)
 
     def declarations(self):
         return self.__declarations
@@ -287,9 +309,13 @@ def __read_location(decl, attrs, to_skip):
         if "name" in attrs and attrs["name"] in to_skip:
             decl.location = declarations.location_t('', -1)
         else:
+            if XML_AN_BEGIN_LINE in attrs:
+                line_number = attrs[XML_AN_BEGIN_LINE]
+            else:
+                line_number = attrs[XML_AN_LINE]
             decl.location = declarations.location_t(
                 file_name=attrs[XML_AN_FILE],
-                line=int(attrs[XML_AN_LINE]))
+                line=int(line_number))
 
     def __update_membership(self, attrs):
         parent = attrs.get(XML_AN_CONTEXT)
@@ -501,6 +527,8 @@ def __read_calldef(self, calldef, attrs, is_declaration):
             else:
                 calldef.does_throw = True
                 calldef.exceptions = throw_stmt.split()
+        if attrs.get(XML_AN_COMMENT):
+            calldef.comment = attrs.get(XML_AN_COMMENT)
 
     def __read_member_function(self, calldef, attrs, is_declaration):
         self.__read_calldef(calldef, attrs, is_declaration)
@@ -551,6 +579,8 @@ def __read_variable(self, attrs):
                 XML_AN_INIT),
             bits=bits)
         self.__read_byte_offset(decl, attrs)
+        if attrs.get(XML_AN_COMMENT):
+            decl.comment = attrs.get(XML_AN_COMMENT)
         return decl
 
     __read_field = __read_variable  # just a synonym
@@ -568,6 +598,8 @@ def __read_class_impl(self, class_type, attrs):
             decl.is_abstract = bool(attrs.get(XML_AN_ABSTRACT, False))
         self.__read_byte_size(decl, attrs)
         self.__read_byte_align(decl, attrs)
+        if attrs.get(XML_AN_COMMENT):
+            decl.comment = attrs.get(XML_AN_COMMENT)
         return decl
 
     def __read_class(self, attrs):
@@ -584,6 +616,13 @@ def __read_casting_operator(self, attrs):
         self.__read_member_function(operator, attrs, True)
         return operator
 
+    def __read_comment(self, attrs):
+        comment = self.__decl_factory.create_comment()
+        comment._start_line = int(attrs.get(XML_AN_BEGIN_LINE))
+        comment._end_line = int(attrs.get(XML_AN_END_LINE))
+        self.__read_location(comment, attrs, self.__name_attrs_to_skip)
+        return comment
+
     def __read_constructor(self, attrs):
         constructor = self.__decl_factory.create_constructor()
         self.__read_member_function(constructor, attrs, True)

setup.py

@@ -35,6 +35,8 @@
                 "pygccxml.declarations",
                 "pygccxml.parser",
                 "pygccxml.utils"],
+      install_requires=[
+          "castxml>=0.3.6"],
       extras_require={
           "test": list(requirements_test),
           "docs": list(requirements_docs),

unittests/data/test_comments.hpp

@@ -0,0 +1,23 @@
+// Copyright 2014-2017 Insight Software Consortium.
+// Copyright 2004-2009 Roman Yakovenko.
+// Distributed under the Boost Software License, Version 1.0.
+// See http://www.boost.org/LICENSE_1_0.txt
+
+/** class comment */
+class test
+{
+public:
+  // Non-doc comment before.
+  /** doc comment */
+  // Non-doc comment after.
+  test();
+
+  /// cxx comment
+  /// with multiple lines
+  int hello();
+
+  //! mutable field comment
+  int val1 = 0;
+  /// bit field comment
+  double val2=2;
+};

unittests/test_all.py

@@ -83,6 +83,7 @@
 from . import test_find_noncopyable_vars
 from . import test_hash
 from . import test_null_comparison
+from . import test_comments
 
 testers = [
     pep8_tester,
@@ -157,6 +158,7 @@
     test_find_noncopyable_vars,
     test_hash,
     test_null_comparison,
+    test_comments,
 ]
 
 if platform.system() != 'Windows':

unittests/test_comments.py

@@ -0,0 +1,65 @@
+# Copyright 2014-2020 Insight Software Consortium.
+# Copyright 2004-2009 Roman Yakovenko.
+# Distributed under the Boost Software License, Version 1.0.
+# See http://www.boost.org/LICENSE_1_0.txt
+
+import unittest
+
+from . import parser_test_case
+
+from pygccxml import parser
+from pygccxml import declarations
+
+
+class Test(parser_test_case.parser_test_case_t):
+    global_ns = None
+
+    def __init__(self, *args):
+        parser_test_case.parser_test_case_t.__init__(self, *args)
+        self.header = "test_comments.hpp"
+        self.global_ns = None
+        self.config.castxml_epic_version = 1
+
+    def setUp(self):
+
+        if not self.global_ns:
+            decls = parser.parse([self.header], self.config)
+            Test.global_ns = declarations.get_global_namespace(decls)
+            Test.xml_generator_from_xml_file = \
+                self.config.xml_generator_from_xml_file
+        self.xml_generator_from_xml_file = Test.xml_generator_from_xml_file
+        self.global_ns = Test.global_ns
+
+    def test(self):
+        """
+        Check the comment parsing
+        """
+        if self.config.castxml_epic_version != 1:
+            # Run this test only with castxml epic version == 1
+            return
+        tclass = self.global_ns.class_("test")
+        self.assertIn("comment", dir(tclass))
+        self.assertEqual(["/** class comment */"], tclass.comment.text)
+
+        tmethod = tclass.member_functions()[0]
+
+        self.assertIn("comment", dir(tmethod))
+        self.assertEqual(["  /// cxx comment", "  /// with multiple lines"], tmethod.comment.text)
+
+        tconstructor = tclass.constructors()[0]
+
+        self.assertIn("comment", dir(tconstructor))
+        self.assertEqual(["  /** doc comment */"], tconstructor.comment.text)
+
+def create_suite():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(Test))
+    return suite
+
+
+def run_suite():
+    unittest.TextTestRunner(verbosity=2).run(create_suite())
+
+
+if __name__ == "__main__":
+    run_suite()