From 38910807501d496c85c1afc1a8c1e9a915b2a1ed Mon Sep 17 00:00:00 2001 From: "Henrik G. Olsson" Date: Fri, 7 Nov 2025 13:34:39 -0800 Subject: [PATCH 1/4] [utils] add support for expected-expansion to update-verify-tests `expected-expansion` can be a bit unergonomic to use, because it requires pointing out not only the line, but also the column (which is not always obvious), and the nested diagnostics have to refer to absolute lines that aren't present in the source file. This makes both creating and updating these test cases easier through automation. --- .../Inputs/unstringify-macro.swift | 19 ++ .../Utils/update-verify-tests/expansion.swift | 240 +++++++++++++++ utils/update_verify_tests/core.py | 280 ++++++++++++++---- 3 files changed, 482 insertions(+), 57 deletions(-) create mode 100644 test/Utils/update-verify-tests/Inputs/unstringify-macro.swift create mode 100644 test/Utils/update-verify-tests/expansion.swift diff --git a/test/Utils/update-verify-tests/Inputs/unstringify-macro.swift b/test/Utils/update-verify-tests/Inputs/unstringify-macro.swift new file mode 100644 index 000000000000..e5c33f86c43d --- /dev/null +++ b/test/Utils/update-verify-tests/Inputs/unstringify-macro.swift @@ -0,0 +1,19 @@ +import SwiftSyntax +import SwiftSyntaxBuilder +import SwiftSyntaxMacros + +public struct UnstringifyPeerMacro: PeerMacro { + public static func expansion( + of node: AttributeSyntax, + providingPeersOf declaration: some DeclSyntaxProtocol, + in context: some MacroExpansionContext + ) throws -> [DeclSyntax] { + do { + let argumentList = node.arguments!.as(LabeledExprListSyntax.self)! + let arguments = [LabeledExprSyntax](argumentList) + let arg = arguments.first!.expression.as(StringLiteralExprSyntax.self)! + let content = arg.representedLiteralValue! + return [DeclSyntax("\(raw: content)")] + } + } +} diff --git a/test/Utils/update-verify-tests/expansion.swift b/test/Utils/update-verify-tests/expansion.swift new file mode 100644 index 000000000000..db1f80a5678a --- /dev/null +++ b/test/Utils/update-verify-tests/expansion.swift @@ -0,0 +1,240 @@ +// REQUIRES: swift_swift_parser, executable_test + +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +// Building this macro takes some time, so amortise the cost by using it for multiple sub tests +// RUN: %host-build-swift -swift-version 5 -emit-library -o %t/%target-library-name(UnstringifyMacroDefinition) -module-name=UnstringifyMacroDefinition \ +// RUN: %S/Inputs/unstringify-macro.swift -g -no-toolchain-stdlib-rpath + + + +// RUN: not %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/single.swift 2>%t/output.txt +// RUN: %update-verify-tests < %t/output.txt +// RUN: %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/single.swift +// RUN: %diff %t/single.swift %t/single.swift.expected + +// RUN: not %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/multiple.swift 2>%t/output.txt +// RUN: %update-verify-tests < %t/output.txt +// RUN: %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/multiple.swift +// RUN: %diff %t/multiple.swift %t/multiple.swift.expected + +// RUN: not %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/existing.swift 2>%t/output.txt +// RUN: %update-verify-tests < %t/output.txt +// RUN: %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/existing.swift +// RUN: %diff %t/existing.swift %t/existing.swift.expected + +// RUN: not %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/gone.swift 2>%t/output.txt +// RUN: %update-verify-tests < %t/output.txt +// RUN: %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/gone.swift +// RUN: %diff %t/gone.swift %t/gone.swift.expected + +// RUN: not %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/wrong-location.swift 2>%t/output.txt +// RUN: %update-verify-tests < %t/output.txt +// RUN: %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/wrong-location.swift +// RUN: %diff %t/wrong-location.swift %t/wrong-location.swift.expected + +// RUN: not %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/nested.swift 2>%t/output.txt +// RUN: %update-verify-tests < %t/output.txt +// RUN: %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/nested.swift +// RUN: %diff %t/nested.swift %t/nested.swift.expected + +//--- single.swift +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +@unstringifyPeer(""" +func foo(_ x: Int) { + let a = x +} +""") +func foo() {} + +//--- single.swift.expected +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +// expected-note@+1{{in expansion of macro 'unstringifyPeer' on global function 'foo()' here}} +@unstringifyPeer(""" +func foo(_ x: Int) { + let a = x +} +""") +// expected-expansion@+3:14{{ +// expected-warning@2{{initialization of immutable value 'a' was never used; consider replacing with assignment to '_' or removing it}} +// }} +func foo() {} + +//--- multiple.swift +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +@unstringifyPeer(""" +func foo(_ x: Int) { + a = 2 + b = x +} +""") +func foo() {} + +//--- multiple.swift.expected +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +// expected-note@+1 4{{in expansion of macro 'unstringifyPeer' on global function 'foo()' here}} +@unstringifyPeer(""" +func foo(_ x: Int) { + a = 2 + b = x +} +""") +// expected-expansion@+5:14{{ +// expected-note@1 2{{'x' declared here}} +// expected-error@2{{cannot find 'a' in scope; did you mean 'x'?}} +// expected-error@3{{cannot find 'b' in scope; did you mean 'x'?}} +// }} +func foo() {} + +//--- existing.swift +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +@unstringifyPeer(""" +func foo(_ x: Int) { + a = 2 + b = x +} +""") + //expected-expansion@+4:14{{ + // expected-note@1 {{'x' declared here}} + // expected-error@3 {{cannot find 'b' in scope; did you mean 'x'?}} + //}} +func foo() {} + +//--- existing.swift.expected +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +// expected-note@+1 4{{in expansion of macro 'unstringifyPeer' on global function 'foo()' here}} +@unstringifyPeer(""" +func foo(_ x: Int) { + a = 2 + b = x +} +""") + //expected-expansion@+5:14{{ + // expected-error@3 {{cannot find 'b' in scope; did you mean 'x'?}} + // expected-note@1 2{{'x' declared here}} + // expected-error@2 {{cannot find 'a' in scope; did you mean 'x'?}} + //}} +func foo() {} + +//--- gone.swift + //expected-expansion@+4:14{{ + // expected-note@1 {{'x' declared here}} + // expected-error@3 {{cannot find 'b' in scope; did you mean 'x'?}} + //}} +func foo() {} + +//--- gone.swift.expected +func foo() {} + +//--- wrong-location.swift +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + + // expected-expansion@2:14{{ + // expected-note@2 {{'x' declared here}} + // expected-error@3 {{cannot find 'b' in scope; did you mean 'x'?}} + // }} +@unstringifyPeer(""" +func foo(_ x: Int) { + a = 2 + b = x +} +""") +func foo() {} + +//--- wrong-location.swift.expected +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +// expected-note@+1 4{{in expansion of macro 'unstringifyPeer' on global function 'foo()' here}} +@unstringifyPeer(""" +func foo(_ x: Int) { + a = 2 + b = x +} +""") +// expected-expansion@+5:14{{ +// expected-note@1 2{{'x' declared here}} +// expected-error@2{{cannot find 'a' in scope; did you mean 'x'?}} +// expected-error@3{{cannot find 'b' in scope; did you mean 'x'?}} +// }} +func foo() {} + +//--- nested.swift +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") +// hack to make this seem non-recursive +@attached(peer, names: overloaded) +macro unstringifyPeer2(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +@unstringifyPeer(""" +func bar(_ y: Int) { + @unstringifyPeer2(\""" + func foo(_ x: Int) { + a = 2 + b = x + } + \""") + func foo() {} + foo(y) +} +""") +func bar() {} + +//--- nested.swift.expected +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") +// hack to make this seem non-recursive +@attached(peer, names: overloaded) +macro unstringifyPeer2(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +// expected-note@+1 7{{in expansion of macro 'unstringifyPeer' on global function 'bar()' here}} +@unstringifyPeer(""" +func bar(_ y: Int) { + @unstringifyPeer2(\""" + func foo(_ x: Int) { + a = 2 + b = x + } + \""") + func foo() {} + foo(y) +} +""") +// expected-expansion@+10:14{{ +// expected-note@1 2{{did you mean 'y'?}} +// expected-note@2 4{{in expansion of macro 'unstringifyPeer2' on local function 'foo()' here}} +// expected-expansion@9:6{{ +// expected-note@1 2{{did you mean 'x'?}} +// expected-error@2{{cannot find 'a' in scope}} +// expected-error@3{{cannot find 'b' in scope}} +// }} +// expected-error@10{{argument passed to call that takes no arguments}} +// }} +func bar() {} + diff --git a/utils/update_verify_tests/core.py b/utils/update_verify_tests/core.py index 20ad70d393b9..093486d70ed9 100644 --- a/utils/update_verify_tests/core.py +++ b/utils/update_verify_tests/core.py @@ -62,11 +62,13 @@ def __init__( category, parsed_target_line_n, line_is_absolute, + col, count, line, is_re, whitespace_strings, is_from_source_file, + nested_lines, ): self.prefix = prefix self.diag_content = diag_content @@ -80,6 +82,10 @@ def __init__( self.absolute_target() self.whitespace_strings = whitespace_strings self.is_from_source_file = is_from_source_file + self.col = col + self.nested_lines = nested_lines + self.parent = None + self.closer = None def decrement_count(self): self.count -= 1 @@ -142,35 +148,64 @@ def render(self): if self.whitespace_strings: whitespace1_s = self.whitespace_strings[0] whitespace2_s = self.whitespace_strings[1] - whitespace3_s = self.whitespace_strings[2] else: whitespace1_s = " " whitespace2_s = "" - whitespace3_s = "" if count_s and not whitespace2_s: whitespace2_s = " " # required to parse correctly elif not count_s and whitespace2_s == " ": """Don't emit a weird extra space. However if the whitespace is something other than the standard single space, let it be to avoid disrupting manual formatting. - The existence of a non-empty whitespace2_s implies this was parsed with - a count > 1 and then decremented, otherwise this whitespace would have - been parsed as whitespace3_s. """ whitespace2_s = "" - return f"//{whitespace1_s}expected-{self.prefix}{self.category}{re_s}{line_location_s}{whitespace2_s}{count_s}{whitespace3_s}{{{{{self.diag_content}}}}}" + col_s = f":{self.col}" if self.col and (self.category == "expansion" or self.is_from_source_file) else "" + base_s = f"//{whitespace1_s}expected-{self.prefix}{self.category}{re_s}{line_location_s}{col_s}{whitespace2_s}{count_s}" + if self.category == "expansion": + return base_s + "{{" + else: + return base_s + "{{" + self.diag_content + "}}" + + +class ExpansionDiagClose: + def __init__(self, whitespace, line): + self.whitespace = whitespace + self.line = line + self.parent = None + self.category = "closing" + + def render(self): + return "//"+self.whitespace+"}}" expected_diag_re = re.compile( - r"//(\s*)expected-([a-zA-Z-]*)(note|warning|error)(-re)?(@[+-]?\d+)?(?:(\s*)(\d+))?(\s*)\{\{(.*)\}\}" + r"//(\s*)expected-([a-zA-Z-]*)(note|warning|error)(-re)?(@[+-]?\d+)?(:\d+)?(\s*)(\d+)?\{\{(.*)\}\}" +) +expected_expansion_diag_re = re.compile( + r"//(\s*)expected-([a-zA-Z-]*)(expansion)(-re)?(@[+-]?\d+)(:\d+)(\s*)(\d+)?\{\{(.*)" +) +expected_expansion_close_re = re.compile( + r"//(\s*)\}\}" ) def parse_diag(line, filename, prefix): s = line.content ms = expected_diag_re.findall(s) + matched_re = expected_diag_re if not ms: - return None + ms = expected_expansion_diag_re.findall(s) + matched_re = expected_expansion_diag_re + if not ms: + ms = expected_expansion_close_re.findall(s) + if not ms: + return None + if len(ms) > 1: + raise KnownException( + f"multiple closed scopes on line {filename}:{line.line_n}. Aborting due to missing implementation." + ) + line.content = expected_expansion_close_re.sub("{{DIAG}}", s) + return ExpansionDiagClose(ms[0], line) if len(ms) > 1: raise KnownException( f"multiple diags on line {filename}:{line.line_n}. Aborting due to missing implementation." @@ -181,9 +216,9 @@ def parse_diag(line, filename, prefix): category_s, re_s, target_line_s, + target_col_s, whitespace2_s, count_s, - whitespace3_s, diag_s, ] = ms[0] if check_prefix != prefix and check_prefix != "": @@ -200,8 +235,9 @@ def parse_diag(line, filename, prefix): else: target_line_n = int(target_line_s[1:]) is_absolute = True + col = int(target_col_s[1:]) if target_col_s else None count = int(count_s) if count_s else 1 - line.content = expected_diag_re.sub("{{DIAG}}", s) + line.content = matched_re.sub("{{DIAG}}", s) return Diag( check_prefix, @@ -209,11 +245,13 @@ def parse_diag(line, filename, prefix): category_s, target_line_n, is_absolute, + col, count, line, bool(re_s), - [whitespace1_s, whitespace2_s, whitespace3_s], + [whitespace1_s, whitespace2_s], True, + [], ) @@ -246,9 +284,7 @@ def orig_line_n_to_new_line_n(line_n, orig_lines): return orig_lines[line_n - 1].line_n -def add_diag(orig_line_n, diag_s, diag_category, lines, orig_lines, prefix): - line_n = orig_line_n_to_new_line_n(orig_line_n, orig_lines) - target = lines[line_n - 1] +def infer_line_context(target, line_n): for other in target.targeting_diags: if other.is_re: raise KnownException( @@ -278,30 +314,68 @@ def add_diag(orig_line_n, diag_s, diag_category, lines, orig_lines, prefix): else: new_line_n = prev_line.line_n assert new_line_n == line_n + (not reverse) - total_offset + return (prev_line, total_offset, new_line_n) + + +def add_diag(orig_target_line_n, col, diag_s, diag_category, lines, orig_lines, prefix, nested_context): + if nested_context: + prev_line = None + for line in lines: + if line.diag and line.diag.absolute_target() < orig_target_line_n: + prev_line = line + if prev_line: + new_line_n = prev_line.line_n + 1 + else: + prev_line = nested_context.line + new_line_n = 1 + else: + line_n = orig_line_n_to_new_line_n(orig_target_line_n, orig_lines) + target = lines[line_n - 1] - new_line = Line(get_indent(prev_line.content) + "{{DIAG}}\n", new_line_n) + prev_line, total_offset, new_line_n = infer_line_context(target, line_n) + indent = get_indent(prev_line.content) + new_line = Line(indent + "{{DIAG}}\n", new_line_n) add_line(new_line, lines) - whitespace_strings = prev_line.diag.whitespace_strings if prev_line.diag else None + whitespace_strings = None + if prev_line.diag: + whitespace_strings = prev_line.diag.whitespace_strings.copy() if prev_line.diag.whitespace_strings else None + if prev_line.diag == nested_context: + if not whitespace_strings: + whitespace_strings = [" ", "", ""] + whitespace_strings[0] += " " + new_diag = Diag( prefix, diag_s, diag_category, - total_offset, - False, + orig_target_line_n if nested_context else total_offset, + bool(nested_context), + col, 1, new_line, False, whitespace_strings, False, + [], ) new_line.diag = new_diag - new_diag.set_target(target) + if not nested_context: + new_diag.set_target(target) + return new_diag def remove_dead_diags(lines): - for line in lines: - if not line.diag or line.diag.count != 0: + for line in lines.copy(): + if not line.diag: + continue + if line.diag.category == "expansion": + remove_dead_diags(line.diag.nested_lines) + if line.diag.nested_lines: + line.diag.count = 1 + else: + line.diag.count = 0 + if line.diag.count != 0: continue if line.render() == "": remove_line(line, lines) @@ -320,55 +394,115 @@ def remove_dead_diags(lines): remove_line(other_diag.line, lines) -def update_test_file(filename, diag_errors, prefix, updated_test_files): - dprint(f"updating test file {filename}") - if filename in updated_test_files: - raise KnownException(f"{filename} already updated, but got new output") +def fold_expansions(lines): + i = 0 + while i < len(lines): + line = lines[i] + if not line.diag or not line.diag.parent: + i += 1 + continue + remove_line(line, lines) + if line.diag.category == "closing": + line.diag.parent.closer = line + else: + line.line_n = len(line.diag.parent.nested_lines) + add_line(line, line.diag.parent.nested_lines) + + +def expand_expansions(lines): + i = 0 + while i < len(lines): + line = lines[i] + if not line.diag or line.diag.category != "expansion": + i += 1 + continue + for j, nested in enumerate(line.diag.nested_lines + [line.diag.closer]): + nested.line_n = line.line_n + j + 1 + add_line(nested, lines) + i += 1 + + +def error_refers_to_diag(diag_error, diag, target_line_n): + if diag_error.col and diag.col and diag_error.col != diag.col: + return False + return target_line_n == diag.absolute_target() and diag_error.category == diag.category and (diag.category == "expansion" or diag_error.content == diag.diag_content) + + +def find_other_targeting(lines, orig_lines, is_nested, diag_error): + if is_nested: + other_diags = [line.diag for line in lines if line.diag and error_refers_to_diag(diag_error, line.diag, diag_error.line)] else: - updated_test_files.add(filename) - with open(filename, "r") as f: - lines = [Line(line, i + 1) for i, line in enumerate(f.readlines() + [''])] - orig_lines = list(lines) + target = orig_lines[diag_error.line - 1] + other_diags = [ + d + for d in target.targeting_diags + if error_refers_to_diag(diag_error, d, target.line_n) + ] + return other_diags - for line in lines: - diag = parse_diag(line, filename, prefix) - if diag: - line.diag = diag - diag.set_target(lines[diag.absolute_target() - 1]) +def update_lines(diag_errors, lines, orig_lines, prefix, filename, nested_context): for diag_error in diag_errors: if not isinstance(diag_error, NotFoundDiag): continue - # this is a diagnostic expected but not seen line_n = diag_error.line - assert lines[line_n - 1].diag - if not lines[line_n - 1].diag or diag_error.content != lines[line_n - 1].diag.diag_content: + line = orig_lines[line_n - 1] + assert line.diag or nested_context + if not line.diag or diag_error.content != line.diag.diag_content: raise KnownException( - f"{filename}:{line_n} - found diag {lines[line_n - 1].diag.diag_content} but expected {diag_error.content}" + f"{filename}:{line_n} - found diag {line.diag.diag_content} but expected {diag_error.content}" ) - if diag_error.category != lines[line_n - 1].diag.category: + if diag_error.category != line.diag.category: raise KnownException( - f"{filename}:{line_n} - found {lines[line_n - 1].diag.category} diag but expected {diag_error.category}" + f"{filename}:{line_n} - found {line.diag.category} diag but expected {diag_error.category}" ) - lines[line_n - 1].diag.decrement_count() + line.diag.decrement_count() - diag_errors.sort(reverse=True, key=lambda t: t.line) + diag_errors.sort(reverse=True, key=lambda diag_error: diag_error.line) for diag_error in diag_errors: - if not isinstance(diag_error, ExtraDiag): + if not isinstance(diag_error, ExtraDiag) and not isinstance(diag_error, NestedDiag): continue - line_n = diag_error.line - target = orig_lines[line_n - 1] - other_diags = [ - d - for d in target.targeting_diags - if d.diag_content == diag_error.content and d.category == diag_error.category - ] - other_diag = other_diags[0] if other_diags else None - if other_diag: - other_diag.increment_count() + other_diags = find_other_targeting(lines, orig_lines, bool(nested_context), diag_error) + diag = other_diags[0] if other_diags else None + if diag: + diag.increment_count() else: - add_diag(line_n, diag_error.content, diag_error.category, lines, orig_lines, diag_error.prefix) + diag = add_diag(diag_error.line, diag_error.col, diag_error.content, diag_error.category, lines, orig_lines, diag_error.prefix, nested_context) + if isinstance(diag_error, NestedDiag): + if not diag.closer: + whitespace = diag.whitespace_strings[0] if diag.whitespace_strings else " " + diag.closer = Line(get_indent(diag.line.content) + "//" + whitespace + "}}\n", None) + update_lines([diag_error.nested], diag.nested_lines, orig_lines, prefix, diag_error.file, diag) + + +def update_test_file(filename, diag_errors, prefix, updated_test_files): + dprint(f"updating test file {filename}") + if filename in updated_test_files: + raise KnownException(f"{filename} already updated, but got new output") + else: + updated_test_files.add(filename) + with open(filename, "r") as f: + lines = [Line(line, i + 1) for i, line in enumerate(f.readlines() + [''])] + orig_lines = list(lines) + + expansion_context = [] + for line in lines: + diag = parse_diag(line, filename, prefix) + if diag: + line.diag = diag + if expansion_context: + diag.parent = expansion_context[-1] + else: + diag.set_target(lines[diag.absolute_target() - 1]) + if diag.category == "expansion": + expansion_context.append(diag) + elif diag.category == "closing": + expansion_context.pop() + + fold_expansions(lines) + update_lines(diag_errors, lines, orig_lines, prefix, filename, None) remove_dead_diags(lines) + expand_expansions(lines) with open(filename, "w") as f: for line in lines: f.write(line.render()) @@ -432,6 +566,14 @@ def update_test_files(errors, prefix): """ diag_error_re4 = re.compile(r"(\S+):(\d+):(\d+): error: expected (\S+), not (\S+)") +""" +ex: +test.swift:12:14: note: in expansion from here +func foo() {} + ^ +""" +diag_expansion_note_re = re.compile(r"(\S+):(\d+):(\d+): note: in expansion from here") + class NotFoundDiag: def __init__(self, file, line, col, category, content, prefix): @@ -459,17 +601,36 @@ def __str__(self): return f"{self.file}:{self.line}:{self.col}: error unexpected {self.category} produced: {self.content}" +class NestedDiag: + def __init__(self, file, line, col, nested): + self.file = file + self.line = line + self.col = col + self.category = "expansion" + self.content = None + self.nested = nested + self.prefix = "" + + def __str__(self): + return f""" +{self.file}:{self.line}:{self.col}: note: in expansion from here ( + {self.nested} +) +""" + + def check_expectations(tool_output, prefix): """ The entry point function. Called by the stand-alone update-verify-tests.py as well as litplugin.py. """ - curr = [] + top_level = [] try: i = 0 while i < len(tool_output): line = tool_output[i].strip() + curr = [] if not "error:" in line: pass elif m := diag_error_re.match(line): @@ -500,10 +661,15 @@ def check_expectations(tool_output, prefix): dprint(line.strip()) i += 1 + while curr and i < len(tool_output) and (m := diag_expansion_note_re.match(tool_output[i].strip())): + curr = [NestedDiag(m.group(1), int(m.group(2)), int(m.group(3)), e) for e in curr] + i += 3 + top_level.extend(curr) + except KnownException as e: return (1, f"Error in update-verify-tests while parsing tool output: {e}") - if curr: - return (0, update_test_files(curr, prefix)) + if top_level: + return (0, update_test_files(top_level, prefix)) else: return (1, "no mismatching diagnostics found") From d71c8cc29fc7933a64bed1113665180912a6858f Mon Sep 17 00:00:00 2001 From: "Henrik G. Olsson" Date: Fri, 7 Nov 2025 13:50:05 -0800 Subject: [PATCH 2/4] [utils] run the formatter on update-verify-tests (NFC) --- utils/update-verify-tests.py | 5 +- utils/update_verify_tests/core.py | 176 +++++++++++++++++++++++++----- 2 files changed, 147 insertions(+), 34 deletions(-) diff --git a/utils/update-verify-tests.py b/utils/update-verify-tests.py index de7c5fdeb60c..22c89d6a9730 100644 --- a/utils/update-verify-tests.py +++ b/utils/update-verify-tests.py @@ -30,9 +30,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--prefix", default="", help="The prefix passed to -verify" - ) + parser.add_argument("--prefix", default="", help="The prefix passed to -verify") args = parser.parse_args() (ret_code, output) = check_expectations(sys.stdin.readlines(), args.prefix) print(output) @@ -41,4 +39,3 @@ def main(): if __name__ == "__main__": main() - diff --git a/utils/update_verify_tests/core.py b/utils/update_verify_tests/core.py index 093486d70ed9..ea4f5e642aca 100644 --- a/utils/update_verify_tests/core.py +++ b/utils/update_verify_tests/core.py @@ -159,7 +159,11 @@ def render(self): standard single space, let it be to avoid disrupting manual formatting. """ whitespace2_s = "" - col_s = f":{self.col}" if self.col and (self.category == "expansion" or self.is_from_source_file) else "" + col_s = ( + f":{self.col}" + if self.col and (self.category == "expansion" or self.is_from_source_file) + else "" + ) base_s = f"//{whitespace1_s}expected-{self.prefix}{self.category}{re_s}{line_location_s}{col_s}{whitespace2_s}{count_s}" if self.category == "expansion": return base_s + "{{" @@ -175,7 +179,7 @@ def __init__(self, whitespace, line): self.category = "closing" def render(self): - return "//"+self.whitespace+"}}" + return "//" + self.whitespace + "}}" expected_diag_re = re.compile( @@ -184,9 +188,7 @@ def render(self): expected_expansion_diag_re = re.compile( r"//(\s*)expected-([a-zA-Z-]*)(expansion)(-re)?(@[+-]?\d+)(:\d+)(\s*)(\d+)?\{\{(.*)" ) -expected_expansion_close_re = re.compile( - r"//(\s*)\}\}" -) +expected_expansion_close_re = re.compile(r"//(\s*)\}\}") def parse_diag(line, filename, prefix): @@ -317,7 +319,16 @@ def infer_line_context(target, line_n): return (prev_line, total_offset, new_line_n) -def add_diag(orig_target_line_n, col, diag_s, diag_category, lines, orig_lines, prefix, nested_context): +def add_diag( + orig_target_line_n, + col, + diag_s, + diag_category, + lines, + orig_lines, + prefix, + nested_context, +): if nested_context: prev_line = None for line in lines: @@ -339,7 +350,11 @@ def add_diag(orig_target_line_n, col, diag_s, diag_category, lines, orig_lines, whitespace_strings = None if prev_line.diag: - whitespace_strings = prev_line.diag.whitespace_strings.copy() if prev_line.diag.whitespace_strings else None + whitespace_strings = ( + prev_line.diag.whitespace_strings.copy() + if prev_line.diag.whitespace_strings + else None + ) if prev_line.diag == nested_context: if not whitespace_strings: whitespace_strings = [" ", "", ""] @@ -425,12 +440,21 @@ def expand_expansions(lines): def error_refers_to_diag(diag_error, diag, target_line_n): if diag_error.col and diag.col and diag_error.col != diag.col: return False - return target_line_n == diag.absolute_target() and diag_error.category == diag.category and (diag.category == "expansion" or diag_error.content == diag.diag_content) + return ( + target_line_n == diag.absolute_target() + and diag_error.category == diag.category + and (diag.category == "expansion" or diag_error.content == diag.diag_content) + ) def find_other_targeting(lines, orig_lines, is_nested, diag_error): if is_nested: - other_diags = [line.diag for line in lines if line.diag and error_refers_to_diag(diag_error, line.diag, diag_error.line)] + other_diags = [ + line.diag + for line in lines + if line.diag + and error_refers_to_diag(diag_error, line.diag, diag_error.line) + ] else: target = orig_lines[diag_error.line - 1] other_diags = [ @@ -460,19 +484,43 @@ def update_lines(diag_errors, lines, orig_lines, prefix, filename, nested_contex diag_errors.sort(reverse=True, key=lambda diag_error: diag_error.line) for diag_error in diag_errors: - if not isinstance(diag_error, ExtraDiag) and not isinstance(diag_error, NestedDiag): + if not isinstance(diag_error, ExtraDiag) and not isinstance( + diag_error, NestedDiag + ): continue - other_diags = find_other_targeting(lines, orig_lines, bool(nested_context), diag_error) + other_diags = find_other_targeting( + lines, orig_lines, bool(nested_context), diag_error + ) diag = other_diags[0] if other_diags else None if diag: diag.increment_count() else: - diag = add_diag(diag_error.line, diag_error.col, diag_error.content, diag_error.category, lines, orig_lines, diag_error.prefix, nested_context) + diag = add_diag( + diag_error.line, + diag_error.col, + diag_error.content, + diag_error.category, + lines, + orig_lines, + diag_error.prefix, + nested_context, + ) if isinstance(diag_error, NestedDiag): if not diag.closer: - whitespace = diag.whitespace_strings[0] if diag.whitespace_strings else " " - diag.closer = Line(get_indent(diag.line.content) + "//" + whitespace + "}}\n", None) - update_lines([diag_error.nested], diag.nested_lines, orig_lines, prefix, diag_error.file, diag) + whitespace = ( + diag.whitespace_strings[0] if diag.whitespace_strings else " " + ) + diag.closer = Line( + get_indent(diag.line.content) + "//" + whitespace + "}}\n", None + ) + update_lines( + [diag_error.nested], + diag.nested_lines, + orig_lines, + prefix, + diag_error.file, + diag, + ) def update_test_file(filename, diag_errors, prefix, updated_test_files): @@ -482,7 +530,7 @@ def update_test_file(filename, diag_errors, prefix, updated_test_files): else: updated_test_files.add(filename) with open(filename, "r") as f: - lines = [Line(line, i + 1) for i, line in enumerate(f.readlines() + [''])] + lines = [Line(line, i + 1) for i, line in enumerate(f.readlines() + [""])] orig_lines = list(lines) expansion_context = [] @@ -544,7 +592,9 @@ def update_test_files(errors, prefix): a = 2 ^ """ -diag_error_re2 = re.compile(r"(\S+):(\d+):(\d+): error: unexpected (\S+) produced: (.*)") +diag_error_re2 = re.compile( + r"(\S+):(\d+):(\d+): error: unexpected (\S+) produced: (.*)" +) """ @@ -634,35 +684,102 @@ def check_expectations(tool_output, prefix): if not "error:" in line: pass elif m := diag_error_re.match(line): - diag = parse_diag(Line(tool_output[i+1], int(m.group(2))), m.group(1), prefix) + diag = parse_diag( + Line(tool_output[i + 1], int(m.group(2))), m.group(1), prefix + ) i += 2 - curr.append(NotFoundDiag(m.group(1), int(m.group(2)), int(m.group(3)), m.group(4), diag.diag_content, diag.prefix)) + curr.append( + NotFoundDiag( + m.group(1), + int(m.group(2)), + int(m.group(3)), + m.group(4), + diag.diag_content, + diag.prefix, + ) + ) elif m := diag_error_re2.match(line): - curr.append(ExtraDiag(m.group(1), int(m.group(2)), int(m.group(3)), m.group(4), m.group(5), prefix)) + curr.append( + ExtraDiag( + m.group(1), + int(m.group(2)), + int(m.group(3)), + m.group(4), + m.group(5), + prefix, + ) + ) i += 2 # Create two mirroring mismatches when the compiler reports that the category or diagnostic is incorrect. # This makes it easier to handle cases where the same diagnostic is mentioned both in an incorrect message/category # diagnostic, as well as in an error not produced diagnostic. This can happen for things like 'expected-error 2{{foo}}' # if only one diagnostic is emitted on that line, and the content of that diagnostic is actually 'bar'. elif m := diag_error_re3.match(line): - diag = parse_diag(Line(tool_output[i+1], int(m.group(2))), m.group(1), prefix) - curr.append(NotFoundDiag(m.group(1), int(m.group(2)), int(m.group(3)), diag.category, diag.diag_content, diag.prefix)) - curr.append(ExtraDiag(m.group(1), diag.absolute_target(), int(m.group(3)), diag.category, tool_output[i+3].strip(), diag.prefix)) + diag = parse_diag( + Line(tool_output[i + 1], int(m.group(2))), m.group(1), prefix + ) + curr.append( + NotFoundDiag( + m.group(1), + int(m.group(2)), + int(m.group(3)), + diag.category, + diag.diag_content, + diag.prefix, + ) + ) + curr.append( + ExtraDiag( + m.group(1), + diag.absolute_target(), + int(m.group(3)), + diag.category, + tool_output[i + 3].strip(), + diag.prefix, + ) + ) i += 3 elif m := diag_error_re4.match(line): - diag = parse_diag(Line(tool_output[i+1], int(m.group(2))), m.group(1), prefix) + diag = parse_diag( + Line(tool_output[i + 1], int(m.group(2))), m.group(1), prefix + ) assert diag.category == m.group(4) - assert tool_output[i+3].strip() == m.group(5) - curr.append(NotFoundDiag(m.group(1), int(m.group(2)), int(m.group(3)), diag.category, diag.diag_content, diag.prefix)) - curr.append(ExtraDiag(m.group(1), diag.absolute_target(), int(m.group(3)), m.group(5), diag.diag_content, diag.prefix)) + assert tool_output[i + 3].strip() == m.group(5) + curr.append( + NotFoundDiag( + m.group(1), + int(m.group(2)), + int(m.group(3)), + diag.category, + diag.diag_content, + diag.prefix, + ) + ) + curr.append( + ExtraDiag( + m.group(1), + diag.absolute_target(), + int(m.group(3)), + m.group(5), + diag.diag_content, + diag.prefix, + ) + ) i += 3 else: dprint("no match") dprint(line.strip()) i += 1 - while curr and i < len(tool_output) and (m := diag_expansion_note_re.match(tool_output[i].strip())): - curr = [NestedDiag(m.group(1), int(m.group(2)), int(m.group(3)), e) for e in curr] + while ( + curr + and i < len(tool_output) + and (m := diag_expansion_note_re.match(tool_output[i].strip())) + ): + curr = [ + NestedDiag(m.group(1), int(m.group(2)), int(m.group(3)), e) + for e in curr + ] i += 3 top_level.extend(curr) @@ -672,4 +789,3 @@ def check_expectations(tool_output, prefix): return (0, update_test_files(top_level, prefix)) else: return (1, "no mismatching diagnostics found") - From bf46369de7846a992a53d37aa65168d6d77ba827 Mon Sep 17 00:00:00 2001 From: "Henrik G. Olsson" Date: Fri, 7 Nov 2025 16:02:05 -0800 Subject: [PATCH 3/4] [utils] only use column for comparison when emitted This fixes a bug that was introduced where two diagnostics on the same line, with the same content, would be emitted separately if they occurred on separate columns. This despite the fact that neither of these checks specify the column when emitted. The checks are now properly merged again. --- utils/update_verify_tests/core.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/utils/update_verify_tests/core.py b/utils/update_verify_tests/core.py index ea4f5e642aca..6ab365b080ff 100644 --- a/utils/update_verify_tests/core.py +++ b/utils/update_verify_tests/core.py @@ -82,7 +82,7 @@ def __init__( self.absolute_target() self.whitespace_strings = whitespace_strings self.is_from_source_file = is_from_source_file - self.col = col + self._col = col self.nested_lines = nested_lines self.parent = None self.closer = None @@ -116,6 +116,12 @@ def absolute_target(self): def relative_target(self): return self.absolute_target() - self.line.line_n + def col(self): + # expected-expansion requires column. Otherwise only retain column info if it's already there. + if self._col and (self.category == "expansion" or self.is_from_source_file): + return self._col + return None + def take(self, other_diag): assert self.count == 0 assert other_diag.count > 0 @@ -159,11 +165,7 @@ def render(self): standard single space, let it be to avoid disrupting manual formatting. """ whitespace2_s = "" - col_s = ( - f":{self.col}" - if self.col and (self.category == "expansion" or self.is_from_source_file) - else "" - ) + col_s = f":{self.col()}" if self.col() else "" base_s = f"//{whitespace1_s}expected-{self.prefix}{self.category}{re_s}{line_location_s}{col_s}{whitespace2_s}{count_s}" if self.category == "expansion": return base_s + "{{" @@ -438,7 +440,7 @@ def expand_expansions(lines): def error_refers_to_diag(diag_error, diag, target_line_n): - if diag_error.col and diag.col and diag_error.col != diag.col: + if diag_error.col and diag.col() and diag_error.col != diag.col(): return False return ( target_line_n == diag.absolute_target() From f1b504bc11db89e56b44b36613d980618a33333e Mon Sep 17 00:00:00 2001 From: "Henrik G. Olsson" Date: Fri, 7 Nov 2025 19:59:32 -0800 Subject: [PATCH 4/4] [utils] integrate update-verify-tests with lit's --update-tests This adds a lit plugin wrapping the core of update-verify-tests. When lit is invoked with --update-tests it will call the plugin, which checks if the failure is due to a -verify mismatch. If it is, it tries to repair the test. If the source file was originally created by split-file, the changes are propagated to the parent file. No tests are added, because I don't want to run nested llvm-lit tests in Swift's test suite, but the core functionality is tested through update-verify-tests.py. --- test/lit.cfg | 5 + utils/update-verify-tests.py | 11 +- utils/update_verify_tests/core.py | 16 +-- utils/update_verify_tests/litplugin.py | 135 +++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 11 deletions(-) create mode 100644 utils/update_verify_tests/litplugin.py diff --git a/test/lit.cfg b/test/lit.cfg index 65aa8d8cb8fc..0b251588f60a 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -3291,3 +3291,8 @@ lit_config.note(f"Target Triple: {config.target_triple}, Variant Triple: {config lit_config.note("Available features: " + ", ".join(sorted(config.available_features))) config.substitutions.append( ('%use_no_opaque_pointers', '-Xcc -Xclang -Xcc -no-opaque-pointers' ) ) + +if lit_config.update_tests: + sys.path.append(config.swift_utils) + from update_verify_tests.litplugin import uvt_lit_plugin + lit_config.test_updaters.append(uvt_lit_plugin) diff --git a/utils/update-verify-tests.py b/utils/update-verify-tests.py index 22c89d6a9730..93e789ee6984 100644 --- a/utils/update-verify-tests.py +++ b/utils/update-verify-tests.py @@ -32,9 +32,14 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--prefix", default="", help="The prefix passed to -verify") args = parser.parse_args() - (ret_code, output) = check_expectations(sys.stdin.readlines(), args.prefix) - print(output) - sys.exit(ret_code) + (err, updated_files) = check_expectations(sys.stdin.readlines(), args.prefix) + if err: + print(err) + sys.exit(1) + + if len(updated_files) > 1: + print("\n\t".join(["updated files:"] + updated_files)) + print(f"updated file: {updated_files[0]}") if __name__ == "__main__": diff --git a/utils/update_verify_tests/core.py b/utils/update_verify_tests/core.py index 6ab365b080ff..2ee63881e952 100644 --- a/utils/update_verify_tests/core.py +++ b/utils/update_verify_tests/core.py @@ -570,13 +570,13 @@ def update_test_files(errors, prefix): try: update_test_file(filename, diag_errors, prefix, updated_test_files) except KnownException as e: - return f"Error in update-verify-tests while updating {filename}: {e}" + return ( + f"Error in update-verify-tests while updating {filename}: {e}", + None, + ) updated_files = list(updated_test_files) assert updated_files - if len(updated_files) == 1: - return f"updated file {updated_files[0]}" - updated_files_s = "\n\t".join(updated_files) - return "updated files:\n\t{updated_files_s}" + return (None, updated_files) """ @@ -786,8 +786,8 @@ def check_expectations(tool_output, prefix): top_level.extend(curr) except KnownException as e: - return (1, f"Error in update-verify-tests while parsing tool output: {e}") + return (f"Error in update-verify-tests while parsing tool output: {e}", None) if top_level: - return (0, update_test_files(top_level, prefix)) + return update_test_files(top_level, prefix) else: - return (1, "no mismatching diagnostics found") + return ("no mismatching diagnostics found", None) diff --git a/utils/update_verify_tests/litplugin.py b/utils/update_verify_tests/litplugin.py new file mode 100644 index 000000000000..1769a68e6ddc --- /dev/null +++ b/utils/update_verify_tests/litplugin.py @@ -0,0 +1,135 @@ +import os +import shlex +import pathlib +from update_verify_tests.core import check_expectations + +""" +This file provides the `uvt_lit_plugin` function, which is invoked on failed RUN lines when lit is executed with --update-tests. +It checks whether the failed command is a swift compiler invocation with the `-verify` flag and analyses the output to try to +repair the failed test. If the updated file was originally created by `split-file` it updates the corresponding slice in the source file. +""" + + +class SplitFileTarget: + def __init__(self, slice_start_idx, test_path, lines, name): + self.slice_start_idx = slice_start_idx + self.test_path = test_path + self.lines = lines + self.name = name + + def copyFrom(self, source): + lines_before = self.lines[: self.slice_start_idx + 1] + self.lines = self.lines[self.slice_start_idx + 1 :] + slice_end_idx = None + for i, l in enumerate(self.lines): + if SplitFileTarget._get_split_line_path(l) != None: + slice_end_idx = i + break + if slice_end_idx is not None: + lines_after = self.lines[slice_end_idx:] + else: + lines_after = [] + with open(source, "r") as f: + new_lines = lines_before + f.readlines() + lines_after + with open(self.test_path, "w") as f: + for l in new_lines: + f.write(l) + + def __str__(self): + return f"slice {self.name} in {self.test_path}" + + @staticmethod + def get_target_dir(commands, test_path): + # posix=True breaks Windows paths because \ is treated as an escaping character + for cmd in commands: + split = shlex.split(cmd, posix=False) + if "split-file" not in split: + continue + start_idx = split.index("split-file") + split = split[start_idx:] + if len(split) < 3: + continue + p = unquote(split[1].strip()) + if not test_path.samefile(p): + continue + return unquote(split[2].strip()) + return None + + @staticmethod + def create(path, commands, test_path, target_dir): + path = pathlib.Path(path) + with open(test_path, "r") as f: + lines = f.readlines() + for i, l in enumerate(lines): + p = SplitFileTarget._get_split_line_path(l) + if p and path.samefile(os.path.join(target_dir, p)): + idx = i + break + else: + return None + return SplitFileTarget(idx, test_path, lines, p) + + @staticmethod + def _get_split_line_path(l): + if len(l) < 6: + return None + if l.startswith("//"): + l = l[2:] + else: + l = l[1:] + if l.startswith("--- "): + l = l[4:] + else: + return None + return l.rstrip() + + +def unquote(s): + if len(s) > 1 and s[0] == s[-1] and (s[0] == '"' or s[0] == "'"): + return s[1:-1] + return s + + +def propagate_split_files(test_path, updated_files, commands): + test_path = pathlib.Path(test_path) + split_target_dir = SplitFileTarget.get_target_dir(commands, test_path) + if not split_target_dir: + return updated_files + + new = [] + for file in updated_files: + target = SplitFileTarget.create(file, commands, test_path, split_target_dir) + if target: + target.copyFrom(file) + new.append(target) + else: + new.append(file) + return new + + +def uvt_lit_plugin(result, test, commands): + if ( + not any(e.endswith("swift-frontend") for e in result.command.args) + or not "-verify" in result.command.args + ): + return None + + prefix = "" + for i, arg in enumerate(result.command.args): + if arg == "-verify-additional-prefix": + if i + 1 >= len(result.command.args): + return None + if prefix: + # can only handle at most 1 additional prefix at the moment + return None + prefix = result.command.args[i + 1] + + (err, updated_files) = check_expectations(result.stderr.split("\n"), prefix) + if err: + return err + + updated_files = propagate_split_files(test.getFilePath(), updated_files, commands) + + if len(updated_files) > 1: + return "\n\t".join(["updated files:"] + updated_files) + return f"updated file: {updated_files[0]}"