Swift: fix subtle codegen bug on missing files

While the internal registry was being cleaned up from files removed by
codegen itself, it was not dropping files removed outside of codegen.

Because of this files removed by the user were not being regenerated
again if no change was staged to them, unless `--force` was provided.

This also fixes some such "ghost" entries in the registry and some
missing generated files.

Author: redsun82

swift/codegen/lib/render.py

@@ -106,7 +106,6 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         if exc_val is None:
             for f in self._existing - self._skipped - self.written:
-                self._hashes.pop(self._get_path(f), None)
                 f.unlink(missing_ok=True)
                 log.info(f"removed {f.name}")
             for f in self.written:
@@ -116,6 +115,10 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             # so that they get the chance to be regenerated again during the next run
             for f in self.written:
                 self._hashes.pop(self._get_path(f), None)
+        # clean up the registry from files that do not exist any more
+        for f in list(self._hashes):
+            if not (self._swift_dir / f).exists():
+                self._hashes.pop(f)
         self._dump_registry()
 
     def _do_write(self, mnemonic: str, contents: str, output: pathlib.Path):

swift/codegen/test/test_render.py

@@ -214,23 +214,40 @@ class MyError(Exception):
     pass
 
 
-def test_managed_render_exception_drops_written_from_registry(pystache_renderer, sut):
+def test_managed_render_exception_drops_written_and_inexsistent_from_registry(pystache_renderer, sut):
     data = mock.Mock(spec=("template",))
     text = "some text"
     pystache_renderer.render_name.side_effect = (text,)
     output = paths.swift_dir / "some/output.txt"
-    registry = paths.swift_dir / "a/registry.list"
+    registry = paths.swift_dir / "x/registry.list"
     write(output, text)
+    write(paths.swift_dir / "a")
+    write(paths.swift_dir / "c")
     write(registry, "a a a\n"
                     f"some/output.txt whatever {hash(text)}\n"
-                    "b b b")
+                    "b b b\n"
+                    "c c c")
 
     with pytest.raises(MyError):
         with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
             renderer.render(data, output)
             raise MyError
 
-    assert_file(registry, "a a a\nb b b\n")
+    assert_file(registry, "a a a\nc c c\n")
+
+
+def test_managed_render_drops_inexsistent_from_registry(pystache_renderer, sut):
+    registry = paths.swift_dir / "x/registry.list"
+    write(paths.swift_dir / "a")
+    write(paths.swift_dir / "c")
+    write(registry, f"a {hash('')} {hash('')}\n"
+                    "b b b\n"
+                    f"c {hash('')} {hash('')}")
+
+    with sut.manage(generated=(), stubs=(), registry=registry):
+        pass
+
+    assert_file(registry, f"a {hash('')} {hash('')}\nc {hash('')} {hash('')}\n")
 
 
 def test_managed_render_exception_does_not_erase(pystache_renderer, sut):

swift/ql/.generated.list

@@ -800,10 +800,6 @@ ql/test/extractor-tests/generated/expr/InterpolatedStringLiteralExpr/MISSING_SOU
 ql/test/extractor-tests/generated/expr/IsExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
 ql/test/extractor-tests/generated/expr/KeyPathApplicationExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
 ql/test/extractor-tests/generated/expr/KeyPathDotExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
-ql/test/extractor-tests/generated/expr/KeyPathExpr/KeyPathExpr.ql 088e9955681f87045f77ce1acd8ccf24d9a418c0b200dc8936511776dda8c8dd a35eb69033c53fd4a986d02b10ec52ac192ea8a5ce7479f191427a329b4704df
-ql/test/extractor-tests/generated/expr/KeyPathExpr/KeyPathExpr_getParsedPath.ql 5f88eaba63c02a42c6ed2093cba9d442f5a3595a5e0888312f51e196c0471cfc b30b8ca1894143a2e0c334e9c32b29c50d13ef5bc94d62d338e019d9e0caeca5
-ql/test/extractor-tests/generated/expr/KeyPathExpr/KeyPathExpr_getRoot.ql 61d8d0f50c62e6bdf98005609861f6f4fd16e59c439706abf03ba27f87ed3cb1 403ee884bb83b7a4207993afbda7964e676f5f64923ce11e65a0cf8bd199e01d
-ql/test/extractor-tests/generated/expr/KeyPathExpr/KeyPathExpr_getType.ql 992497671107be454ffe1f42b513a5bca37bd31849587ad55f6bd87d8ac5d4a7 b51109f0d9e5e6238d8ab9e67f24d435a873a7884308c4f01ec4ecad51ed031d
 ql/test/extractor-tests/generated/expr/KeyPathExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
 ql/test/extractor-tests/generated/expr/LazyInitializerExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd
 ql/test/extractor-tests/generated/expr/MagicIdentifierLiteralExpr/MISSING_SOURCE.txt 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd 7e714762ffb48c436102027d560fb5addc1f7dc6dd6936b06e0d3cca031d67fd

swift/ql/test/extractor-tests/generated/OtherAvailabilitySpec/MISSING_SOURCE.txt

@@ -0,0 +1,4 @@
+// generated by codegen/codegen.py
+
+After a swift source file is added in this directory and codegen/codegen.py is run again, test queries
+will appear and this file will be deleted

swift/ql/test/extractor-tests/generated/PlatformVersionAvailabilitySpec/MISSING_SOURCE.txt

@@ -0,0 +1,4 @@
+// generated by codegen/codegen.py
+
+After a swift source file is added in this directory and codegen/codegen.py is run again, test queries
+will appear and this file will be deleted