[test] Verify usage of swift_feature_ markers without enabling features

When a feature is removed from `Features.def` or when a test removes all
usages of `-enable-experimental/upcoming-feature` keeping the
`REQUIRES:` lines around is probably a mistake. Warn about possibly
missing flags if a `REQUIRES:` is found referencing a feature without
the equivalent `-enable-*-feature` in a `RUN:` line.

Author: drodriguez

test/Misc/verify-swift-feature-testing.test-sh

@@ -18,6 +18,9 @@ EXCEPTIONAL_FILES = [
 FEATURE_USAGE_RE = re.compile(
     r"-enable-(?:experimental|upcoming)-feature (?:-Xfrontend )?([A-Za-z0-9]*)"
 )
+# Be careful of not using REQUIRES or RUN with a colon after them or Lit will
+# pick them up.
+FEATURE_LIT_MARKER_RE = re.compile(r"swift_feature_([A-Za-z0-9]*)")
 
 
 def find_test_files_with_features_usage(swift_src_root):
@@ -38,6 +41,24 @@ def find_test_files_with_features_usage(swift_src_root):
     return output.splitlines()
 
 
+def find_test_files_with_marker_usage(swift_src_root):
+    # Look for every test file in the test directories with `REQUIRES` lines
+    # that mention `swift_feature_`.
+    # Be careful of not using REQUIRES with a colon after them or Lit will
+    # pick them up.
+    output = subprocess.check_output([
+        "grep",
+        "--extended-regexp",
+        "--recursive",
+        "-e",
+        "REQUIRES[:].*swift_feature_",
+        "--files-with-matches",
+        str(swift_src_root / "test"),
+        str(swift_src_root / "validation-test"),
+    ], text=True)
+    return output.splitlines()
+
+
 def find_run_lines(test_file):
     output = subprocess.check_output([
         "grep",
@@ -50,6 +71,18 @@ def find_run_lines(test_file):
     return output.splitlines()
 
 
+def find_requires_lines(test_file):
+    output = subprocess.check_output([
+        "grep",
+        "--extended-regexp",
+        "--no-filename",
+        "-e",
+        "REQUIRES[:]",
+        str(test_file),
+    ], text=True)
+    return output.splitlines()
+
+
 def check_existing_requires(test_file, feature):
     returncode = subprocess.call([
         "grep",
@@ -62,6 +95,21 @@ def check_existing_requires(test_file, feature):
     return returncode != 0
 
 
+def check_existing_feature_usage(test_file, feature):
+    returncode = subprocess.call([
+        "grep",
+        "--extended-regexp",
+        "--quiet",
+        "-e",
+        (
+            "RUN[:].*-enable-(experimental|upcoming)-feature (-Xfrontend )?"
+            + re.escape(feature)
+        ),
+        str(test_file),
+    ])
+    return returncode != 0
+
+
 def check_existing_error_message_checks(test_file, feature):
     returncode = subprocess.call([
         "grep",
@@ -103,6 +151,30 @@ def check_test_file_feature_usage(test_file):
     return num_failures == 0
 
 
+def check_test_file_marker_usage(test_file):
+    require_lines = find_requires_lines(test_file)
+    features = set(
+        feature
+        for line in require_lines
+        for feature in FEATURE_LIT_MARKER_RE.findall(line)
+    )
+    num_failures = 0
+    for feature in features:
+        # No warning if -enable-experimental/upcoming-feature is there
+        if not check_existing_feature_usage(test_file, feature):
+            continue
+
+        # For everything else, print a warning and add to the invalid exit code
+        print(
+            "error: {}: Missing '-enable-experimental/upcoming-feature: {}'".format(
+                str(test_file),
+                feature
+            )
+        )
+        num_failures += 1
+    return num_failures == 0
+
+
 def main():
     if len(sys.argv) < 2:
         print('Invalid number of arguments.')
@@ -121,6 +193,17 @@ def main():
         if not check_test_file_feature_usage(test_file):
             num_failures += 1
 
+    test_files_with_marker_usage = find_test_files_with_marker_usage(swift_src_root)
+    for test_file in test_files_with_marker_usage:
+        test_file = pathlib.Path(test_file)
+
+        # First lets check this is not one of the exceptional files
+        if test_file.relative_to(swift_src_root) in EXCEPTIONAL_FILES:
+            continue
+
+        if not check_test_file_marker_usage(test_file):
+            num_failures += 1
+
     if num_failures > 0:
         sys.exit(1)