Swift: add --force to codegen

Author: redsun82

swift/codegen/codegen.py

@@ -42,6 +42,8 @@ def _parse_args() -> argparse.Namespace:
                    help="output directory for generated C++ files, required if trap or cpp is provided to --generate")
     p.add_argument("--generated-registry", type=_abspath, default=paths.swift_dir / "ql/.generated.list",
                    help="registry file containing information about checked-in generated code")
+    p.add_argument("--force", "-f", action="store_true",
+                   help="generate all files without skipping unchanged files and overwriting modified ones")
     return p.parse_args()
 
 

swift/codegen/generators/qlgen.py

@@ -302,7 +302,8 @@ def generate(opts, renderer):
 
     imports = {}
 
-    with renderer.manage(generated=generated, stubs=stubs, registry=opts.generated_registry) as renderer:
+    with renderer.manage(generated=generated, stubs=stubs, registry=opts.generated_registry,
+                         force=opts.force) as renderer:
 
         db_classes = [cls for cls in classes.values() if not cls.ipa]
         renderer.render(ql.DbClasses(db_classes), out / "Raw.qll")

swift/codegen/lib/render.py

@@ -59,8 +59,8 @@ def _do_write(self, mnemonic: str, contents: str, output: pathlib.Path):
         log.debug(f"{mnemonic}: generated {output.name}")
 
     def manage(self, generated: typing.Iterable[pathlib.Path], stubs: typing.Iterable[pathlib.Path],
-               registry: pathlib.Path) -> "RenderManager":
-        return RenderManager(self._swift_dir, generated, stubs, registry)
+               registry: pathlib.Path, force: bool = False) -> "RenderManager":
+        return RenderManager(self._swift_dir, generated, stubs, registry, force)
 
 
 class RenderManager(Renderer):
@@ -87,9 +87,10 @@ class Hashes:
 
     def __init__(self, swift_dir: pathlib.Path, generated: typing.Iterable[pathlib.Path],
                  stubs: typing.Iterable[pathlib.Path],
-                 registry: pathlib.Path):
+                 registry: pathlib.Path, force: bool = False):
         super().__init__(swift_dir)
         self._registry_path = registry
+        self._force = force
         self._hashes = {}
         self.written = set()
         self._existing = set()
@@ -132,10 +133,13 @@ def _process_generated(self, generated: typing.Iterable[pathlib.Path]):
         for f in generated:
             self._existing.add(f)
             rel_path = self._get_path(f)
-            if rel_path not in self._hashes:
+            if self._force:
+                pass
+            elif rel_path not in self._hashes:
                 log.warning(f"{rel_path} marked as generated but absent from the registry")
             elif self._hashes[rel_path].post != self._hash_file(f):
-                raise Error(f"{rel_path} is generated but was modified, please revert the file")
+                raise Error(f"{rel_path} is generated but was modified, please revert the file "
+                            "or pass --force to overwrite")
 
     def _process_stubs(self, stubs: typing.Iterable[pathlib.Path]):
         for f in stubs:
@@ -144,10 +148,13 @@ def _process_stubs(self, stubs: typing.Iterable[pathlib.Path]):
                 self._hashes.pop(rel_path, None)
                 continue
             self._existing.add(f)
-            if rel_path not in self._hashes:
+            if self._force:
+                pass
+            elif rel_path not in self._hashes:
                 log.warning(f"{rel_path} marked as stub but absent from the registry")
             elif self._hashes[rel_path].post != self._hash_file(f):
-                raise Error(f"{rel_path} is a stub marked as generated, but it was modified")
+                raise Error(f"{rel_path} is a stub marked as generated, but it was modified, "
+                            "please remove the `// generated` header, revert the file or pass --force to overwrite it")
 
     @staticmethod
     def is_customized_stub(file: pathlib.Path) -> bool:
@@ -171,6 +178,8 @@ def _hash_string(data: str) -> str:
         return h.hexdigest()
 
     def _load_registry(self):
+        if self._force:
+            return
         try:
             with open(self._registry_path) as reg:
                 for line in reg:

swift/codegen/test/test_qlgen.py

@@ -48,6 +48,7 @@ def qlgen_opts(opts):
     opts.generated_registry = generated_registry_path()
     opts.ql_format = True
     opts.swift_dir = paths.swift_dir
+    opts.force = False
     return opts
 
 
@@ -430,7 +431,9 @@ def test_format_error(opts, generate, render_manager, run_mock):
         generate([schema.Class('A')])
 
 
-def test_manage_parameters(opts, generate, renderer):
+@pytest.mark.parametrize("force", [False, True])
+def test_manage_parameters(opts, generate, renderer, force):
+    opts.force = force
     ql_a = opts.ql_output / "A.qll"
     ql_b = opts.ql_output / "B.qll"
     stub_a = opts.ql_stub_output / "A.qll"
@@ -448,7 +451,7 @@ def test_manage_parameters(opts, generate, renderer):
     generate([schema.Class('A')])
     assert renderer.mock_calls == [
         mock.call.manage(generated={ql_a, ql_b, test_a, test_b, import_file()}, stubs={stub_a, stub_b},
-                         registry=opts.generated_registry)
+                         registry=opts.generated_registry, force=force)
     ]
 
 

swift/codegen/test/test_render.py

@@ -75,6 +75,7 @@ def test_managed_render(pystache_renderer, sut):
         mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
     ]
 
+
 def test_managed_render_with_no_registry(pystache_renderer, sut):
     data = mock.Mock(spec=("template",))
     text = "some text"
@@ -266,5 +267,70 @@ def test_render_with_extensions(pystache_renderer, sut):
         assert_file(expected_output, expected_contents)
 
 
+def test_managed_render_with_force_not_skipping_generated_file(pystache_renderer, sut):
+    data = mock.Mock(spec=("template",))
+    output = paths.swift_dir / "some/output.txt"
+    some_output = "some output"
+    registry = paths.swift_dir / "a/registry.list"
+    write(output, some_output)
+    write(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
+
+    pystache_renderer.render_name.side_effect = (some_output,)
+
+    with sut.manage(generated=(output,), stubs=(), registry=registry, force=True) as renderer:
+        renderer.render(data, output)
+        assert renderer.written == {output}
+        assert_file(output, some_output)
+
+    assert_file(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
+    assert pystache_renderer.mock_calls == [
+        mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
+    ]
+
+
+def test_managed_render_with_force_not_skipping_stub_file(pystache_renderer, sut):
+    data = mock.Mock(spec=("template",))
+    stub = paths.swift_dir / "some/stub.txt"
+    some_output = "// generated some output"
+    some_processed_output = "// generated some processed output"
+    registry = paths.swift_dir / "a/registry.list"
+    write(stub, some_processed_output)
+    write(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
+
+    pystache_renderer.render_name.side_effect = (some_output,)
+
+    with sut.manage(generated=(), stubs=(stub,), registry=registry, force=True) as renderer:
+        renderer.render(data, stub)
+        assert renderer.written == {stub}
+        assert_file(stub, some_output)
+
+    assert_file(registry, f"some/stub.txt {hash(some_output)} {hash(some_output)}\n")
+    assert pystache_renderer.mock_calls == [
+        mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
+    ]
+
+
+def test_managed_render_with_force_ignores_modified_generated_file(sut):
+    output = paths.swift_dir / "some/output.txt"
+    some_processed_output = "// some processed output"
+    registry = paths.swift_dir / "a/registry.list"
+    write(output, "// something else")
+    write(registry, f"some/output.txt whatever {hash(some_processed_output)}\n")
+
+    with sut.manage(generated=(output,), stubs=(), registry=registry, force=True):
+        pass
+
+
+def test_managed_render_with_force_ignores_modified_stub_file_still_marked_as_generated(sut):
+    stub = paths.swift_dir / "some/stub.txt"
+    some_processed_output = "// generated some processed output"
+    registry = paths.swift_dir / "a/registry.list"
+    write(stub, "// generated something else")
+    write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")
+
+    with sut.manage(generated=(), stubs=(stub,), registry=registry, force=True):
+        pass
+
+
 if __name__ == '__main__':
     sys.exit(pytest.main([__file__] + sys.argv[1:]))

swift/ql/.generated.list

@@ -385,7 +385,7 @@ ql/lib/codeql/swift/generated/Synth.qll 90df85be365c89c3c2e22041ee7dc9dd2ad9194b
 ql/lib/codeql/swift/generated/SynthConstructors.qll 5c91f09bd82728651ed61f498704e0f62847788fa986dec5e674d81f294076c7 5c91f09bd82728651ed61f498704e0f62847788fa986dec5e674d81f294076c7
 ql/lib/codeql/swift/generated/UnknownFile.qll 0fcf9beb8de79440bcdfff4bb6ab3dd139bd273e6c32754e05e6a632651e85f6 0fcf9beb8de79440bcdfff4bb6ab3dd139bd273e6c32754e05e6a632651e85f6
 ql/lib/codeql/swift/generated/UnknownLocation.qll e50efefa02a0ec1ff635a00951b5924602fc8cab57e5756e4a039382c69d3882 e50efefa02a0ec1ff635a00951b5924602fc8cab57e5756e4a039382c69d3882
-ql/lib/codeql/swift/generated/UnspecifiedElement.qll a3a73f53c492adc6655fb88b40b1bd9b2c9d365fc5c019b3233c01d6110fb3f2 ba77cd5272cffd1d3aad8bea69786b97aec0c93a4b59a070d621fe2d21c2e90c
+ql/lib/codeql/swift/generated/UnspecifiedElement.qll dbc6ca4018012977b26ca184a88044c55b0661e3998cd14d46295b62a8d69625 184c9a0ce18c2ac881943b0fb400613d1401ed1d5564f90716b6c310ba5afe71
 ql/lib/codeql/swift/generated/decl/AbstractFunctionDecl.qll 8255b24dddda83e8a7dee9d69a4cf9883b5a7ae43676d7242b5aab5169f68982 407c7d63681fb03ad6cb4ea3c2b04be7ccb5ddbe655a8aec4219eb3799bc36e8
 ql/lib/codeql/swift/generated/decl/AbstractStorageDecl.qll 66147ad36cefce974b4ae0f3e84569bd6742ea2f3e842c3c04e6e5cbd17e7928 ce7c2347e2dfe0b141db103ccb8e56a61d286476c201aebe6a275edd7fca2c0f
 ql/lib/codeql/swift/generated/decl/AbstractTypeParamDecl.qll 1e268b00d0f2dbbd85aa70ac206c5e4a4612f06ba0091e5253483635f486ccf9 5479e13e99f68f1f347283535f8098964f7fd4a34326ff36ad5711b2de1ab0d0

swift/ql/lib/codeql/swift/generated/UnspecifiedElement.qll

@@ -5,9 +5,6 @@ import codeql.swift.elements.Element
 import codeql.swift.elements.ErrorElement
 
 module Generated {
-  /**
-   * bla
-   */
   class UnspecifiedElement extends Synth::TUnspecifiedElement, ErrorElement {
     override string getAPrimaryQlClass() { result = "UnspecifiedElement" }
 

swift/schema.py

@@ -40,7 +40,6 @@ class ErrorElement(Locatable):
 
 @use_for_null
 class UnspecifiedElement(ErrorElement):
-    """bla"""
     parent: optional[Element]
     property: string
     index: optional[int]