Merge pull request #17533 from github/redsun82/codegen-parametrized-pragmas

Codegen: introduce inherited pragmas and move remaining decorations

Author: aibaars

misc/codegen/generators/qlgen.py

@@ -115,7 +115,7 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic
         is_unordered=prop.is_unordered,
         description=prop.description,
         synth=bool(cls.synth) or prop.synth,
-        type_is_hideable=lookup[prop.type].hideable if prop.type in lookup else False,
+        type_is_hideable="ql_hideable" in lookup[prop.type].pragmas if prop.type in lookup else False,
         internal="ql_internal" in prop.pragmas,
     )
     if prop.is_single:
@@ -154,7 +154,6 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic
 
 
 def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]) -> ql.Class:
-    pragmas = {k: True for k in cls.pragmas if k.startswith("qltest")}
     prev_child = ""
     properties = []
     for p in cls.properties:
@@ -170,9 +169,8 @@ def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]) -> q
         properties=properties,
         dir=pathlib.Path(cls.group or ""),
         doc=cls.doc,
-        hideable=cls.hideable,
+        hideable="ql_hideable" in cls.pragmas,
         internal="ql_internal" in cls.pragmas,
-        **pragmas,
     )
 
 
@@ -448,7 +446,8 @@ def generate(opts, renderer):
             for c in data.classes.values():
                 if should_skip_qltest(c, data.classes):
                     continue
-                test_with = data.classes[c.test_with] if c.test_with else c
+                test_with_name = c.pragmas.get("qltest_test_with")
+                test_with = data.classes[test_with_name] if test_with_name else c
                 test_dir = test_out / test_with.group / test_with.name
                 test_dir.mkdir(parents=True, exist_ok=True)
                 if all(f.suffix in (".txt", ".ql", ".actual", ".expected") for f in test_dir.glob("*.*")):

misc/codegen/generators/rusttestgen.py

@@ -60,6 +60,7 @@ def generate(opts, renderer):
             if fn:
                 indent = 4 * " "
                 code = [indent + l for l in code]
-            test_with = schema.classes[cls.test_with] if cls.test_with else cls
+            test_with_name = typing.cast(str, cls.pragmas.get("qltest_test_with"))
+            test_with = schema.classes[test_with_name] if test_with_name else cls
             test = opts.ql_test_output / test_with.group / test_with.name / f"gen_{test_name}.rs"
             renderer.render(TestCode(code="\n".join(code), function=fn), test)

misc/codegen/lib/ql.py

@@ -107,9 +107,6 @@ class Class:
     dir: pathlib.Path = pathlib.Path()
     imports: List[str] = field(default_factory=list)
     import_prefix: Optional[str] = None
-    qltest_skip: bool = False
-    qltest_collapse_hierarchy: bool = False
-    qltest_uncollapse_hierarchy: bool = False
     internal: bool = False
     doc: List[str] = field(default_factory=list)
     hideable: bool = False

misc/codegen/lib/schema.py

@@ -91,11 +91,8 @@ class Class:
     bases: List[str] = field(default_factory=list)
     derived: Set[str] = field(default_factory=set)
     properties: List[Property] = field(default_factory=list)
-    group: str = ""
     pragmas: List[str] | Dict[str, object] = field(default_factory=dict)
     doc: List[str] = field(default_factory=list)
-    hideable: bool = False
-    test_with: Optional[str] = None
 
     def __post_init__(self):
         if not isinstance(self.pragmas, dict):
@@ -118,7 +115,7 @@ def check_types(self, known: typing.Iterable[str]):
             if synth.on_arguments is not None:
                 for t in synth.on_arguments.values():
                     _check_type(t, known)
-        _check_type(self.test_with, known)
+        _check_type(self.pragmas.get("qltest_test_with"), known)
 
     @property
     def synth(self) -> SynthInfo | bool | None:
@@ -127,6 +124,10 @@ def synth(self) -> SynthInfo | bool | None:
     def mark_synth(self):
         self.pragmas.setdefault("synth", True)
 
+    @property
+    def group(self) -> str:
+        return typing.cast(str, self.pragmas.get("group", ""))
+
 
 @dataclass
 class Schema:
@@ -211,3 +212,6 @@ def split_doc(doc):
     while trimmed and not trimmed[0]:
         trimmed.pop(0)
     return trimmed
+
+
+inheritable_pragma_prefix = "_inheritable_pragma_"

misc/codegen/lib/schemadefs.py

@@ -5,6 +5,8 @@
 
 from misc.codegen.lib.schema import Property
 
+_set = set
+
 
 @_dataclass
 class _ChildModifier(_schema.PropertyModifier):
@@ -79,7 +81,7 @@ class _SynthModifier(_schema.PropertyModifier, _Namespace):
     def modify(self, prop: _schema.Property):
         prop.synth = self.synth
 
-    def negate(self) -> "PropertyModifier":
+    def negate(self) -> _schema.PropertyModifier:
         return _SynthModifier(self.name, False)
 
 
@@ -100,14 +102,18 @@ class _ClassPragma(_PragmaBase):
     """ A class pragma.
     For schema classes it acts as a python decorator with `@`.
     """
+    inherited: bool = False
     value: object = None
 
     def __call__(self, cls: type) -> type:
         """ use this pragma as a decorator on classes """
-        # not using hasattr as we don't want to land on inherited pragmas
-        if "_pragmas" not in cls.__dict__:
-            cls._pragmas = {}
-        self._apply(cls._pragmas)
+        if self.inherited:
+            setattr(cls, f"{_schema.inheritable_pragma_prefix}{self.pragma}", self.value)
+        else:
+            # not using hasattr as we don't want to land on inherited pragmas
+            if "_pragmas" not in cls.__dict__:
+                cls._pragmas = {}
+            self._apply(cls._pragmas)
         return cls
 
     def _apply(self, pragmas: _Dict[str, object]) -> None:
@@ -125,7 +131,7 @@ class _Pragma(_ClassPragma, _schema.PropertyModifier):
     def modify(self, prop: _schema.Property):
         self._apply(prop.pragmas)
 
-    def negate(self) -> "PropertyModifier":
+    def negate(self) -> _schema.PropertyModifier:
         return _Pragma(self.pragma, remove=True)
 
     def _apply(self, pragmas: _Dict[str, object]) -> None:
@@ -142,13 +148,14 @@ class _ParametrizedClassPragma(_PragmaBase):
     """
     _pragma_class: _ClassVar[type] = _ClassPragma
 
-    function: _Callable[..., object] = None
+    inherited: bool = False
+    factory: _Callable[..., object] = None
 
     def __post_init__(self):
-        self.__signature__ = _inspect.signature(self.function).replace(return_annotation=self._pragma_class)
+        self.__signature__ = _inspect.signature(self.factory).replace(return_annotation=self._pragma_class)
 
     def __call__(self, *args, **kwargs) -> _pragma_class:
-        return self._pragma_class(self.pragma, value=self.function(*args, **kwargs))
+        return self._pragma_class(self.pragma, self.inherited, value=self.factory(*args, **kwargs))
 
 
 @_dataclass
@@ -204,15 +211,6 @@ def __getitem__(self, item):
 _ClassDecorator = _Callable[[type], type]
 
 
-def _annotate(**kwargs) -> _ClassDecorator:
-    def f(cls: type) -> type:
-        for k, v in kwargs.items():
-            setattr(cls, f"_{k}", v)
-        return cls
-
-    return f
-
-
 boolean = "boolean"
 int = "int"
 string = "string"
@@ -226,31 +224,29 @@ def f(cls: type) -> type:
 doc = _DocModifier
 desc = _DescModifier
 
-use_for_null = _annotate(null=True)
+use_for_null = _ClassPragma("null")
 
 qltest.add(_Pragma("skip"))
 qltest.add(_ClassPragma("collapse_hierarchy"))
 qltest.add(_ClassPragma("uncollapse_hierarchy"))
-qltest.test_with = lambda cls: _annotate(test_with=cls)  # inheritable
+qltest.add(_ParametrizedClassPragma("test_with", inherited=True, factory=_schema.get_type_name))
 
-ql.add(_ParametrizedClassPragma("default_doc_name", lambda doc: doc))
-ql.hideable = _annotate(hideable=True)  # inheritable
+ql.add(_ParametrizedClassPragma("default_doc_name", factory=lambda doc: doc))
+ql.add(_ClassPragma("hideable", inherited=True))
 ql.add(_Pragma("internal"))
 
 cpp.add(_Pragma("skip"))
 
 rust.add(_Pragma("skip_doc_test"))
 
-rust.add(_ParametrizedClassPragma("doc_test_signature", lambda signature: signature))
+rust.add(_ParametrizedClassPragma("doc_test_signature", factory=lambda signature: signature))
 
+group = _ParametrizedClassPragma("group", inherited=True, factory=lambda group: group)
 
-def group(name: str = "") -> _ClassDecorator:
-    return _annotate(group=name)
 
-
-synth.add(_ParametrizedClassPragma("from_class", lambda ref: _schema.SynthInfo(
+synth.add(_ParametrizedClassPragma("from_class", factory=lambda ref: _schema.SynthInfo(
     from_class=_schema.get_type_name(ref))), key="synth")
-synth.add(_ParametrizedClassPragma("on_arguments", lambda **kwargs:
+synth.add(_ParametrizedClassPragma("on_arguments", factory=lambda **kwargs:
                                    _schema.SynthInfo(on_arguments={k: _schema.get_type_name(t) for k, t in kwargs.items()})), key="synth")
 
 
@@ -288,16 +284,11 @@ def decorator(cls: type) -> _PropertyAnnotation:
             raise _schema.Error("Annotation classes must be named _")
         if cls.__doc__ is not None:
             annotated_cls.__doc__ = cls.__doc__
-        old_pragmas = getattr(annotated_cls, "_pragmas", None)
-        new_pragmas = getattr(cls, "_pragmas", {})
-        if old_pragmas:
-            old_pragmas.update(new_pragmas)
-        else:
-            annotated_cls._pragmas = new_pragmas
-        for a, v in cls.__dict__.items():
-            # transfer annotations
-            if a.startswith("_") and not a.startswith("__") and a != "_pragmas":
-                setattr(annotated_cls, a, v)
+        for p, v in cls.__dict__.get("_pragmas", {}).items():
+            _ClassPragma(p, value=v)(annotated_cls)
+        for a in dir(cls):
+            if a.startswith(_schema.inheritable_pragma_prefix):
+                setattr(annotated_cls, a, getattr(cls, a))
         for p, a in cls.__annotations__.items():
             if p in annotated_cls.__annotations__:
                 annotated_cls.__annotations__[p] |= a

misc/codegen/loaders/schemaloader.py

@@ -37,19 +37,23 @@ def _get_class(cls: type) -> schema.Class:
     if cls.__name__ != to_underscore_and_back:
         raise schema.Error(f"Class name must be upper camel-case, without capitalized acronyms, found {cls.__name__} "
                            f"instead of {to_underscore_and_back}")
-    if len({b._group for b in cls.__bases__ if hasattr(b, "_group")}) > 1:
+    if len({g for g in (getattr(b, f"{schema.inheritable_pragma_prefix}group", None)
+                        for b in cls.__bases__) if g}) > 1:
         raise schema.Error(f"Bases with mixed groups for {cls.__name__}")
-    if any(getattr(b, "_null", False) for b in cls.__bases__):
+    pragmas = {
+        # dir and getattr inherit from bases
+        a[len(schema.inheritable_pragma_prefix):]: getattr(cls, a)
+        for a in dir(cls) if a.startswith(schema.inheritable_pragma_prefix)
+    }
+    pragmas |= cls.__dict__.get("_pragmas", {})
+    derived = {d.__name__ for d in cls.__subclasses__()}
+    if "null" in pragmas and derived:
         raise schema.Error(f"Null class cannot be derived")
     return schema.Class(name=cls.__name__,
                         bases=[b.__name__ for b in cls.__bases__ if b is not object],
-                        derived={d.__name__ for d in cls.__subclasses__()},
-                        # getattr to inherit from bases
-                        group=getattr(cls, "_group", ""),
-                        hideable=getattr(cls, "_hideable", False),
-                        test_with=_get_name(getattr(cls, "_test_with", None)),
+                        derived=derived,
+                        pragmas=pragmas,
                         # in the following we don't use `getattr` to avoid inheriting
-                        pragmas=cls.__dict__.get("_pragmas", {}),
                         properties=[
                             a | _PropertyNamer(n)
                             for n, a in cls.__dict__.get("__annotations__", {}).items()
@@ -105,21 +109,23 @@ def fill_is_synth(name: str):
 
 def _fill_hideable_information(classes: typing.Dict[str, schema.Class]):
     """ Update the class map propagating the `hideable` attribute upwards in the hierarchy """
-    todo = [cls for cls in classes.values() if cls.hideable]
+    todo = [cls for cls in classes.values() if "ql_hideable" in cls.pragmas]
     while todo:
         cls = todo.pop()
         for base in cls.bases:
             supercls = classes[base]
-            if not supercls.hideable:
-                supercls.hideable = True
+            if "ql_hideable" not in supercls.pragmas:
+                supercls.pragmas["ql_hideable"] = None
                 todo.append(supercls)
 
 
 def _check_test_with(classes: typing.Dict[str, schema.Class]):
     for cls in classes.values():
-        if cls.test_with is not None and classes[cls.test_with].test_with is not None:
-            raise schema.Error(f"{cls.name} has test_with {cls.test_with} which in turn "
-                               f"has test_with {classes[cls.test_with].test_with}, use that directly")
+        test_with = typing.cast(str, cls.pragmas.get("qltest_test_with"))
+        transitive_test_with = test_with and classes[test_with].pragmas.get("qltest_test_with")
+        if test_with and transitive_test_with:
+            raise schema.Error(f"{cls.name} has test_with {test_with} which in turn "
+                               f"has test_with {transitive_test_with}, use that directly")
 
 
 def load(m: types.ModuleType) -> schema.Schema:
@@ -145,11 +151,11 @@ def load(m: types.ModuleType) -> schema.Schema:
                 f"Only one root class allowed, found second root {name}")
         cls.check_types(known)
         classes[name] = cls
-        if getattr(data, "_null", False):
+        if "null" in cls.pragmas:
+            del cls.pragmas["null"]
             if null is not None:
                 raise schema.Error(f"Null class {null} already defined, second null class {name} not allowed")
             null = name
-            cls.is_null_class = True
 
     _fill_synth_information(classes)
     _fill_hideable_information(classes)

misc/codegen/test/test_cppgen.py

@@ -156,10 +156,10 @@ def test_classes_with_dirs(generate_grouped):
     cbase = cpp.Class(name="CBase")
     assert generate_grouped([
         schema.Class(name="A"),
-        schema.Class(name="B", group="foo"),
-        schema.Class(name="CBase", derived={"C"}, group="bar"),
-        schema.Class(name="C", bases=["CBase"], group="bar"),
-        schema.Class(name="D", group="foo/bar/baz"),
+        schema.Class(name="B", pragmas={"group": "foo"}),
+        schema.Class(name="CBase", derived={"C"}, pragmas={"group": "bar"}),
+        schema.Class(name="C", bases=["CBase"], pragmas={"group": "bar"}),
+        schema.Class(name="D", pragmas={"group": "foo/bar/baz"}),
     ]) == {
         ".": [cpp.Class(name="A", trap_name="As", final=True)],
         "foo": [cpp.Class(name="B", trap_name="Bs", final=True)],