Fix collation script to handle module-level skips

This is meant to detect always-skipped tests by checking for skips
which never appear as non-skips.

However, a module-level skip produces a skip for the module which
doesn't *also* get emitted as a success on the "positive" runs. Since
module-level skips have an empty `classname`, use that to detect them
and treat them slightly differently (by counting them and comparing
against the number of files).

Author: sirosen

scripts/aggregate-pytest-reports.py

@@ -10,28 +10,45 @@ def main():
     args = parser.parse_args()
 
     tests_by_name = defaultdict(dict)
+    skipped_module_counts = defaultdict(int)
     for filename in args.FILES:
         tree = ElementTree.parse(filename)
         root = tree.getroot()
 
         for testcase in root.findall("./testsuite/testcase"):
             classname = testcase.get("classname")
             name = testcase.get("name")
-            nodename = f"{classname.replace('.', '/')}.py::{name}"
 
             skip_node = testcase.find("skipped")
-            if skip_node is not None:
-                if "skipped" not in tests_by_name[nodename]:
-                    tests_by_name[nodename]["skipped"] = True
+
+            if classname:
+                nodename = f"{classname.replace('.', '/')}.py::{name}"
+                if skip_node is not None:
+                    if "skipped" not in tests_by_name[nodename]:
+                        tests_by_name[nodename]["skipped"] = True
+                else:
+                    tests_by_name[nodename]["skipped"] = False
             else:
-                tests_by_name[nodename]["skipped"] = False
+                if skip_node is not None:
+                    skipped_module_counts[name] += 1
+
+    skipped_modules = {
+        modname
+        for modname, count in skipped_module_counts.items()
+        if count == len(args.FILES)
+    }
 
     fail = False
     for nodename, attributes in tests_by_name.items():
         if attributes.get("skipped") is True:
             print(f"ALWAYS SKIPPED: {nodename}")
             fail = True
 
+    if skipped_modules:
+        for modname in skipped_modules:
+            print(f"ALWAYS SKIPPED MODULE: {modname}")
+        fail = True
+
     if fail:
         print("fail")
         sys.exit(1)