Merge pull request github#12205 from github/redsun82/swift-codegen

Swift: fix subtle `codegen` bug on missing files

Author: redsun82

swift/codegen/lib/render.py

@@ -106,7 +106,6 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         if exc_val is None:
             for f in self._existing - self._skipped - self.written:
-                self._hashes.pop(self._get_path(f), None)
                 f.unlink(missing_ok=True)
                 log.info(f"removed {f.name}")
             for f in self.written:
@@ -116,6 +115,10 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             # so that they get the chance to be regenerated again during the next run
             for f in self.written:
                 self._hashes.pop(self._get_path(f), None)
+        # clean up the registry from files that do not exist any more
+        for f in list(self._hashes):
+            if not (self._swift_dir / f).exists():
+                self._hashes.pop(f)
         self._dump_registry()
 
     def _do_write(self, mnemonic: str, contents: str, output: pathlib.Path):

swift/codegen/test/test_render.py

@@ -214,23 +214,40 @@ class MyError(Exception):
     pass
 
 
-def test_managed_render_exception_drops_written_from_registry(pystache_renderer, sut):
+def test_managed_render_exception_drops_written_and_inexsistent_from_registry(pystache_renderer, sut):
     data = mock.Mock(spec=("template",))
     text = "some text"
     pystache_renderer.render_name.side_effect = (text,)
     output = paths.swift_dir / "some/output.txt"
-    registry = paths.swift_dir / "a/registry.list"
+    registry = paths.swift_dir / "x/registry.list"
     write(output, text)
+    write(paths.swift_dir / "a")
+    write(paths.swift_dir / "c")
     write(registry, "a a a\n"
                     f"some/output.txt whatever {hash(text)}\n"
-                    "b b b")
+                    "b b b\n"
+                    "c c c")
 
     with pytest.raises(MyError):
         with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
             renderer.render(data, output)
             raise MyError
 
-    assert_file(registry, "a a a\nb b b\n")
+    assert_file(registry, "a a a\nc c c\n")
+
+
+def test_managed_render_drops_inexsistent_from_registry(pystache_renderer, sut):
+    registry = paths.swift_dir / "x/registry.list"
+    write(paths.swift_dir / "a")
+    write(paths.swift_dir / "c")
+    write(registry, f"a {hash('')} {hash('')}\n"
+                    "b b b\n"
+                    f"c {hash('')} {hash('')}")
+
+    with sut.manage(generated=(), stubs=(), registry=registry):
+        pass
+
+    assert_file(registry, f"a {hash('')} {hash('')}\nc {hash('')} {hash('')}\n")
 
 
 def test_managed_render_exception_does_not_erase(pystache_renderer, sut):

swift/ql/.generated.list

@@ -800,10 +800,6 @@ ql/test/extractor-tests/generated/expr/InterpolatedStringLiteralExpr/MISSING_SOU
 ql/test/extractor-tests/generated/expr/IsExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
 ql/test/extractor-tests/generated/expr/KeyPathApplicationExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
 ql/test/extractor-tests/generated/expr/KeyPathDotExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
-ql/test/extractor-tests/generated/expr/KeyPathExpr/KeyPathExpr.ql 088e9955681f87045f77ce1acd8ccf24d9a418c0b200dc8936511776dda8c8dd a35eb69033c53fd4a986d02b10ec52ac192ea8a5ce7479f191427a329b4704df
-ql/test/extractor-tests/generated/expr/KeyPathExpr/KeyPathExpr_getParsedPath.ql 5f88eaba63c02a42c6ed2093cba9d442f5a3595a5e0888312f51e196c0471cfc b30b8ca1894143a2e0c334e9c32b29c50d13ef5bc94d62d338e019d9e0caeca5
-ql/test/extractor-tests/generated/expr/KeyPathExpr/KeyPathExpr_getRoot.ql 61d8d0f50c62e6bdf98005609861f6f4fd16e59c439706abf03ba27f87ed3cb1 403ee884bb83b7a4207993afbda7964e676f5f64923ce11e65a0cf8bd199e01d
-ql/test/extractor-tests/generated/expr/KeyPathExpr/KeyPathExpr_getType.ql 992497671107be454ffe1f42b513a5bca37bd31849587ad55f6bd87d8ac5d4a7 b51109f0d9e5e6238d8ab9e67f24d435a873a7884308c4f01ec4ecad51ed031d
 ql/test/extractor-tests/generated/expr/KeyPathExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
 ql/test/extractor-tests/generated/expr/LazyInitializerExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
 ql/test/extractor-tests/generated/expr/MagicIdentifierLiteralExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd

swift/ql/test/extractor-tests/generated/OtherAvailabilitySpec/MISSING_SOURCE.txt

@@ -0,0 +1,4 @@
+// generated by codegen/codegen.py
+
+After a swift source file is added in this directory and codegen/codegen.py is run again, test queries
+will appear and this file will be deleted

swift/ql/test/extractor-tests/generated/PlatformVersionAvailabilitySpec/MISSING_SOURCE.txt

@@ -0,0 +1,4 @@
+// generated by codegen/codegen.py
+
+After a swift source file is added in this directory and codegen/codegen.py is run again, test queries
+will appear and this file will be deleted