Merge pull request #10516 from github/redsun82/swift-python-schema

Swift: express the schema in Python

Author: redsun82

swift/.pep8

@@ -0,0 +1,3 @@
+[pep8]
+ignore = E302
+max_line_length = 120

swift/BUILD.bazel

@@ -3,6 +3,18 @@ load("@rules_pkg//:install.bzl", "pkg_install")
 load("//:defs.bzl", "codeql_platform")
 load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles")
 
+filegroup(
+    name = "schema",
+    srcs = ["schema.py"],
+    visibility = ["//swift:__subpackages__"],
+)
+
+filegroup(
+    name = "schema_includes",
+    srcs = glob(["*.dbscheme"]),
+    visibility = ["//swift:__subpackages__"],
+)
+
 pkg_files(
     name = "dbscheme_files",
     srcs = [

swift/codegen/BUILD.bazel

@@ -1,26 +1,16 @@
 load("@swift_codegen_deps//:requirements.bzl", "requirement")
 
-filegroup(
-    name = "schema",
-    srcs = ["schema.yml"],
-    visibility = ["//swift:__subpackages__"],
-)
-
-filegroup(
-    name = "schema_includes",
-    srcs = glob(["*.dbscheme"]),
-    visibility = ["//swift:__subpackages__"],
-)
-
 py_binary(
     name = "codegen",
     srcs = ["codegen.py"],
     data = [
-        ":schema",
-        ":schema_includes",
+        "//swift:schema",
+        "//swift:schema_includes",
         "//swift/codegen/templates:cpp",
         "//swift/codegen/templates:trap",
     ],
     visibility = ["//swift:__subpackages__"],
-    deps = ["//swift/codegen/generators"],
+    deps = [
+        "//swift/codegen/generators",
+    ],
 )

swift/codegen/codegen.py

@@ -23,7 +23,7 @@ def _parse_args() -> argparse.Namespace:
     p.add_argument("--swift-dir", type=_abspath, default=paths.swift_dir,
                    help="the directory that should be regarded as the root of the swift codebase. Used to compute QL "
                         "imports and in some comments (default %(default)s)")
-    p.add_argument("--schema", type=_abspath, default=paths.swift_dir / "codegen/schema.yml",
+    p.add_argument("--schema", type=_abspath, default=paths.swift_dir / "schema.py",
                    help="input schema file (default %(default)s)")
     p.add_argument("--dbscheme", type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme",
                    help="output file for dbscheme generation, input file for trap generation (default %(default)s)")

swift/codegen/generators/cppgen.py

@@ -70,16 +70,16 @@ def _get_class(self, name: str) -> cpp.Class:
         )
 
     def get_classes(self):
-        ret = {pathlib.Path(): []}
+        ret = {'': []}
         for k, cls in self._classmap.items():
-            ret.setdefault(cls.dir, []).append(self._get_class(cls.name))
+            ret.setdefault(cls.group, []).append(self._get_class(cls.name))
         return ret
 
 
 def generate(opts, renderer):
     assert opts.cpp_output
-    processor = Processor(schema.load(opts.schema).classes)
+    processor = Processor(schema.load_file(opts.schema).classes)
     out = opts.cpp_output
     for dir, classes in processor.get_classes().items():
-        include_parent = (dir != pathlib.Path())
-        renderer.render(cpp.ClassList(classes, opts.schema, include_parent), out / dir / "TrapClasses")
+        renderer.render(cpp.ClassList(classes, opts.schema,
+                        include_parent=bool(dir)), out / dir / "TrapClasses")

swift/codegen/generators/dbschemegen.py

@@ -14,12 +14,11 @@
 The type hierarchy will be translated to corresponding `union` declarations.
 """
 
-import pathlib
-
 import inflection
 
 from swift.codegen.lib import schema
 from swift.codegen.lib.dbscheme import *
+from typing import Set, List
 
 log = logging.getLogger(__name__)
 
@@ -35,7 +34,7 @@ def cls_to_dbscheme(cls: schema.Class):
     """ Yield all dbscheme entities needed to model class `cls` """
     if cls.derived:
         yield Union(dbtype(cls.name), (dbtype(c) for c in cls.derived))
-    dir = cls.dir if cls.dir != pathlib.Path() else None
+    dir = pathlib.Path(cls.group) if cls.group else None
     # output a table specific to a class only if it is a leaf class or it has 1-to-1 properties
     # Leaf classes need a table to bind the `@` ids
     # 1-to-1 properties are added to a class specific table
@@ -104,7 +103,7 @@ def generate(opts, renderer):
     input = opts.schema
     out = opts.dbscheme
 
-    data = schema.load(input)
+    data = schema.load_file(input)
 
     dbscheme = Scheme(src=input.relative_to(opts.swift_dir),
                       includes=get_includes(data, include_dir=input.parent, swift_dir=opts.swift_dir),

swift/codegen/generators/qlgen.py

@@ -51,6 +51,10 @@ class RootElementHasChildren(Error):
     pass
 
 
+class NoClasses(Error):
+    pass
+
+
 def get_ql_property(cls: schema.Class, prop: schema.Property, prev_child: str = "") -> ql.Property:
     args = dict(
         type=prop.type if not prop.is_predicate else "predicate",
@@ -103,7 +107,7 @@ def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]):
         bases=cls.bases,
         final=not cls.derived,
         properties=properties,
-        dir=cls.dir,
+        dir=pathlib.Path(cls.group or ""),
         ipa=bool(cls.ipa),
         **pragmas,
     )
@@ -127,7 +131,7 @@ def get_ql_ipa_class_db(name: str) -> ql.Synth.FinalClassDb:
 def get_ql_ipa_class(cls: schema.Class):
     if cls.derived:
         return ql.Synth.NonFinalClass(name=cls.name, derived=sorted(cls.derived),
-                                      root=(cls.name == schema.root_class_name))
+                                      root=not cls.bases)
     if cls.ipa and cls.ipa.from_class is not None:
         source = cls.ipa.from_class
         get_ql_ipa_class_db(source).subtract_type(cls.name)
@@ -253,12 +257,14 @@ def generate(opts, renderer):
     existing |= {q for q in test_out.rglob("*.ql")}
     existing |= {q for q in test_out.rglob(missing_test_source_filename)}
 
-    data = schema.load(input)
+    data = schema.load_file(input)
 
     classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items()}
-    # element root is absent in tests
-    if schema.root_class_name in classes and classes[schema.root_class_name].has_children:
-        raise RootElementHasChildren
+    if not classes:
+        raise NoClasses
+    root = next(iter(classes.values()))
+    if root.has_children:
+        raise RootElementHasChildren(root)
 
     imports = {}
 
@@ -288,10 +294,10 @@ def generate(opts, renderer):
     for c in data.classes.values():
         if _should_skip_qltest(c, data.classes):
             continue
-        test_dir = test_out / c.dir / c.name
+        test_dir = test_out / c.group / c.name
         test_dir.mkdir(parents=True, exist_ok=True)
         if not any(test_dir.glob("*.swift")):
-            log.warning(f"no test source in {c.dir / c.name}")
+            log.warning(f"no test source in {test_dir.relative_to(test_out)}")
             renderer.render(ql.MissingTestInstructions(),
                             test_dir / missing_test_source_filename)
             continue
@@ -308,12 +314,12 @@ def generate(opts, renderer):
     constructor_imports = []
     ipa_constructor_imports = []
     stubs = {}
-    for cls in sorted(data.classes.values(), key=lambda cls: (cls.dir, cls.name)):
+    for cls in sorted(data.classes.values(), key=lambda cls: (cls.group, cls.name)):
         ipa_type = get_ql_ipa_class(cls)
         if ipa_type.is_final:
             final_ipa_types.append(ipa_type)
             if ipa_type.has_params:
-                stub_file = stub_out / cls.dir / f"{cls.name}Constructor.qll"
+                stub_file = stub_out / cls.group / f"{cls.name}Constructor.qll"
                 if not stub_file.is_file() or _is_generated_stub(stub_file):
                     # stub rendering must be postponed as we might not have yet all subtracted ipa types in `ipa_type`
                     stubs[stub_file] = ql.Synth.ConstructorStub(ipa_type)
@@ -326,7 +332,7 @@ def generate(opts, renderer):
 
     for stub_file, data in stubs.items():
         renderer.render(data, stub_file)
-    renderer.render(ql.Synth.Types(schema.root_class_name, final_ipa_types, non_final_ipa_types), out / "Synth.qll")
+    renderer.render(ql.Synth.Types(root.name, final_ipa_types, non_final_ipa_types), out / "Synth.qll")
     renderer.render(ql.ImportList(constructor_imports), out / "SynthConstructors.qll")
     renderer.render(ql.ImportList(ipa_constructor_imports), out / "PureSynthConstructors.qll")
 

swift/codegen/lib/BUILD.bazel

@@ -2,7 +2,7 @@ load("@swift_codegen_deps//:requirements.bzl", "requirement")
 
 py_library(
     name = "lib",
-    srcs = glob(["*.py"]),
+    srcs = glob(["**/*.py"]),
     visibility = ["//swift/codegen:__subpackages__"],
     deps = [
         requirement("pystache"),

swift/codegen/lib/schema.py

@@ -0,0 +1 @@
+from .schema import *