Skip to content

Commit 105adf3

Browse files
ZalatharZalathar
committed
Fix instrument-coverage tests by using Python to sort instantiation groups
1 parent 74c4821 commit 105adf3

File tree

9 files changed

+146
-47
lines changed

9 files changed

+146
-47
lines changed

‎tests/run-make/coverage-reports/Makefile‎

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ endif
138138
) \
139139
2> "$(TMPDIR)"/show_coverage_stderr.$@.txt \
140140
| "$(PYTHON)" $(BASEDIR)/normalize_paths.py \
141+
| "$(PYTHON)" $(BASEDIR)/sort_subviews.py \
141142
> "$(TMPDIR)"/actual_show_coverage.$@.txt || \
142143
( status=$$? ; \
143144
>&2 cat "$(TMPDIR)"/show_coverage_stderr.$@.txt ; \
@@ -158,28 +159,10 @@ ifdef RUSTC_BLESS_TEST
158159
else
159160
# Compare the show coverage output (`--bless` refreshes `typical` files).
160161
#
161-
# FIXME(richkadel): None of the Rust test source samples have the
162-
# `// ignore-llvm-cov-show-diffs` anymore. This directive exists to work around a limitation
163-
# with `llvm-cov show`. When reporting coverage for multiple instantiations of a generic function,
164-
# with different type substitutions, `llvm-cov show` prints these in a non-deterministic order,
165-
# breaking the `diff` comparison.
166-
#
167-
# A partial workaround is implemented below, with `diff --ignore-matching-lines=RE`
168-
# to ignore each line prefixing each generic instantiation coverage code region.
169-
#
170-
# This workaround only works if the coverage counts are identical across all reported
171-
# instantiations. If there is no way to ensure this, you may need to apply the
172-
# `// ignore-llvm-cov-show-diffs` directive, and check for differences using the
173-
# `.json` files to validate that results have not changed. (Until then, the JSON
174-
# files are redundant, so there is no need to generate `expected_*.json` files or
175-
# compare actual JSON results.)
176-
177-
$(DIFF) --ignore-matching-lines='^ \| .*::<.*>.*:$$' --ignore-matching-lines='^ \| <.*>::.*:$$' \
178-
expected_show_coverage.$@.txt "$(TMPDIR)"/actual_show_coverage.$@.txt || \
179-
( grep -q '^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR)/$@.rs && \
180-
>&2 echo 'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \
181-
) || \
182-
( >&2 echo 'diff failed, and not suppressed without `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs'; \
183-
false \
184-
)
162+
# `llvm-cov show` normally prints instantiation groups in an unpredictable
163+
# order, but we have used `sort_subviews.py` to sort them, so we can still
164+
# check the output directly with `diff`.
165+
166+
$(DIFF) expected_show_coverage.$@.txt "$(TMPDIR)"/actual_show_coverage.$@.txt || \
167+
( >&2 echo 'diff failed in $(SOURCEDIR)/$@.rs'; false)
185168
endif

‎tests/run-make/coverage-reports/expected_show_coverage.async.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,9 @@
4141
41| 1| // executed asynchronously.
4242
42| 1| match x {
4343
43| 1| y if c(x).await == y + 1 => { d().await; }
44-
^0 ^0 ^0 ^0
44+
^0 ^0 ^0 ^0
4545
44| 1| y if f().await == y + 1 => (),
46-
^0 ^0 ^0
46+
^0 ^0 ^0
4747
45| 1| _ => (),
4848
46| | }
4949
47| 1|}

‎tests/run-make/coverage-reports/expected_show_coverage.sort_groups.txt‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
1| |// compile-flags: --edition=2021
2+
2| |
3+
3| |// Demonstrate that `sort_subviews.py` can sort instantiation groups into a
4+
4| |// predictable order, while preserving their heterogeneous contents.
5+
5| |
6+
6| 1|fn main() {
7+
7| 1| let cond = std::env::args().len() > 1;
8+
8| 1| generic_fn::<()>(cond);
9+
9| 1| generic_fn::<&'static str>(!cond);
10+
10| 1| if false {
11+
11| 0| generic_fn::<char>(cond);
12+
12| 1| }
13+
13| 1| generic_fn::<i32>(cond);
14+
14| 1| other_fn();
15+
15| 1|}
16+
16| |
17+
17| 3|fn generic_fn<T>(cond: bool) {
18+
18| 3| if cond {
19+
19| 1| println!("{}", std::any::type_name::<T>());
20+
20| 2| }
21+
21| 3|}
22+
------------------
23+
| Unexecuted instantiation: sort_groups::generic_fn::<char>
24+
------------------
25+
| sort_groups::generic_fn::<&str>:
26+
| 17| 1|fn generic_fn<T>(cond: bool) {
27+
| 18| 1| if cond {
28+
| 19| 1| println!("{}", std::any::type_name::<T>());
29+
| 20| 1| }
30+
| ^0
31+
| 21| 1|}
32+
------------------
33+
| sort_groups::generic_fn::<()>:
34+
| 17| 1|fn generic_fn<T>(cond: bool) {
35+
| 18| 1| if cond {
36+
| 19| 0| println!("{}", std::any::type_name::<T>());
37+
| 20| 1| }
38+
| 21| 1|}
39+
------------------
40+
| sort_groups::generic_fn::<i32>:
41+
| 17| 1|fn generic_fn<T>(cond: bool) {
42+
| 18| 1| if cond {
43+
| 19| 0| println!("{}", std::any::type_name::<T>());
44+
| 20| 1| }
45+
| 21| 1|}
46+
------------------
47+
22| |
48+
23| 1|fn other_fn() {}
49+

‎tests/run-make/coverage-reports/expected_show_coverage.uses_crate.txt‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,29 +19,29 @@
1919
18| 2| println!("used_only_from_bin_crate_generic_function with {:?}", arg);
2020
19| 2|}
2121
------------------
22-
| used_crate::used_only_from_bin_crate_generic_function::<&str>:
22+
| Unexecuted instantiation: used_crate::used_only_from_bin_crate_generic_function::<_>
23+
------------------
24+
| used_crate::used_only_from_bin_crate_generic_function::<&alloc::vec::Vec<i32>>:
2325
| 17| 1|pub fn used_only_from_bin_crate_generic_function<T: Debug>(arg: T) {
2426
| 18| 1| println!("used_only_from_bin_crate_generic_function with {:?}", arg);
2527
| 19| 1|}
2628
------------------
27-
| used_crate::used_only_from_bin_crate_generic_function::<&alloc::vec::Vec<i32>>:
29+
| used_crate::used_only_from_bin_crate_generic_function::<&str>:
2830
| 17| 1|pub fn used_only_from_bin_crate_generic_function<T: Debug>(arg: T) {
2931
| 18| 1| println!("used_only_from_bin_crate_generic_function with {:?}", arg);
3032
| 19| 1|}
31-
------------------
32-
| Unexecuted instantiation: used_crate::used_only_from_bin_crate_generic_function::<_>
3333
------------------
3434
20| |// Expect for above function: `Unexecuted instantiation` (see below)
3535
21| 2|pub fn used_only_from_this_lib_crate_generic_function<T: Debug>(arg: T) {
3636
22| 2| println!("used_only_from_this_lib_crate_generic_function with {:?}", arg);
3737
23| 2|}
3838
------------------
39-
| used_crate::used_only_from_this_lib_crate_generic_function::<alloc::vec::Vec<i32>>:
39+
| used_crate::used_only_from_this_lib_crate_generic_function::<&str>:
4040
| 21| 1|pub fn used_only_from_this_lib_crate_generic_function<T: Debug>(arg: T) {
4141
| 22| 1| println!("used_only_from_this_lib_crate_generic_function with {:?}", arg);
4242
| 23| 1|}
4343
------------------
44-
| used_crate::used_only_from_this_lib_crate_generic_function::<&str>:
44+
| used_crate::used_only_from_this_lib_crate_generic_function::<alloc::vec::Vec<i32>>:
4545
| 21| 1|pub fn used_only_from_this_lib_crate_generic_function<T: Debug>(arg: T) {
4646
| 22| 1| println!("used_only_from_this_lib_crate_generic_function with {:?}", arg);
4747
| 23| 1|}
@@ -51,12 +51,12 @@
5151
26| 2| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg);
5252
27| 2|}
5353
------------------
54-
| used_crate::used_from_bin_crate_and_lib_crate_generic_function::<alloc::vec::Vec<i32>>:
54+
| used_crate::used_from_bin_crate_and_lib_crate_generic_function::<&str>:
5555
| 25| 1|pub fn used_from_bin_crate_and_lib_crate_generic_function<T: Debug>(arg: T) {
5656
| 26| 1| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg);
5757
| 27| 1|}
5858
------------------
59-
| used_crate::used_from_bin_crate_and_lib_crate_generic_function::<&str>:
59+
| used_crate::used_from_bin_crate_and_lib_crate_generic_function::<alloc::vec::Vec<i32>>:
6060
| 25| 1|pub fn used_from_bin_crate_and_lib_crate_generic_function<T: Debug>(arg: T) {
6161
| 26| 1| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg);
6262
| 27| 1|}

‎tests/run-make/coverage-reports/expected_show_coverage.uses_inline_crate.txt‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
40| 2| println!("used_only_from_bin_crate_generic_function with {:?}", arg);
4343
41| 2|}
4444
------------------
45+
| Unexecuted instantiation: used_inline_crate::used_only_from_bin_crate_generic_function::<_>
46+
------------------
4547
| used_inline_crate::used_only_from_bin_crate_generic_function::<&alloc::vec::Vec<i32>>:
4648
| 39| 1|pub fn used_only_from_bin_crate_generic_function<T: Debug>(arg: T) {
4749
| 40| 1| println!("used_only_from_bin_crate_generic_function with {:?}", arg);
@@ -51,8 +53,6 @@
5153
| 39| 1|pub fn used_only_from_bin_crate_generic_function<T: Debug>(arg: T) {
5254
| 40| 1| println!("used_only_from_bin_crate_generic_function with {:?}", arg);
5355
| 41| 1|}
54-
------------------
55-
| Unexecuted instantiation: used_inline_crate::used_only_from_bin_crate_generic_function::<_>
5656
------------------
5757
42| |// Expect for above function: `Unexecuted instantiation` (see notes in `used_crate.rs`)
5858
43| |
@@ -77,15 +77,15 @@
7777
51| 3| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg);
7878
52| 3|}
7979
------------------
80-
| used_inline_crate::used_from_bin_crate_and_lib_crate_generic_function::<alloc::vec::Vec<i32>>:
81-
| 50| 1|pub fn used_from_bin_crate_and_lib_crate_generic_function<T: Debug>(arg: T) {
82-
| 51| 1| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg);
83-
| 52| 1|}
84-
------------------
8580
| used_inline_crate::used_from_bin_crate_and_lib_crate_generic_function::<&str>:
8681
| 50| 2|pub fn used_from_bin_crate_and_lib_crate_generic_function<T: Debug>(arg: T) {
8782
| 51| 2| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg);
8883
| 52| 2|}
84+
------------------
85+
| used_inline_crate::used_from_bin_crate_and_lib_crate_generic_function::<alloc::vec::Vec<i32>>:
86+
| 50| 1|pub fn used_from_bin_crate_and_lib_crate_generic_function<T: Debug>(arg: T) {
87+
| 51| 1| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg);
88+
| 52| 1|}
8989
------------------
9090
53| |
9191
54| |#[inline(always)]

‎tests/run-make/coverage-reports/sort_subviews.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env python3
2+
3+
# `llvm-cov show` prints grouped subviews (e.g. for generic functions) in an
4+
# unstable order, which is inconvenient when checking output snapshots with
5+
# `diff`. To work around that, this script detects consecutive subviews in its
6+
# piped input, and sorts them while preserving their contents.
7+
8+
from __future__ import print_function
9+
10+
import sys
11+
12+
13+
def main():
14+
subviews = []
15+
16+
def flush_subviews():
17+
if not subviews:
18+
return
19+
20+
# The last "subview" should be just a boundary line on its own, so
21+
# temporarily remove it before sorting the accumulated subviews.
22+
terminator = subviews.pop()
23+
subviews.sort()
24+
subviews.append(terminator)
25+
26+
for view in subviews:
27+
for line in view:
28+
print(line, end="")
29+
30+
subviews.clear()
31+
32+
for line in sys.stdin:
33+
if line.startswith(" ------------------"):
34+
# This is a subview boundary line, so start a new subview.
35+
subviews.append([line])
36+
elif line.startswith(" |"):
37+
# Add this line to the current subview.
38+
subviews[-1].append(line)
39+
else:
40+
# This line is not part of a subview, so sort and print any
41+
# accumulated subviews, and then print the line as-is.
42+
flush_subviews()
43+
print(line, end="")
44+
45+
flush_subviews()
46+
assert not subviews
47+
48+
49+
if __name__ == "__main__":
50+
main()

‎tests/run-make/coverage/sort_groups.rs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// compile-flags: --edition=2021
2+
3+
// Demonstrate that `sort_subviews.py` can sort instantiation groups into a
4+
// predictable order, while preserving their heterogeneous contents.
5+
6+
fn main() {
7+
let cond = std::env::args().len() > 1;
8+
generic_fn::<()>(cond);
9+
generic_fn::<&'static str>(!cond);
10+
if false {
11+
generic_fn::<char>(cond);
12+
}
13+
generic_fn::<i32>(cond);
14+
other_fn();
15+
}
16+
17+
fn generic_fn<T>(cond: bool) {
18+
if cond {
19+
println!("{}", std::any::type_name::<T>());
20+
}
21+
}
22+
23+
fn other_fn() {}

‎tests/run-make/coverage/uses_crate.rs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// FIXME #110395
2-
// ignore-llvm-cov-show-diffs
3-
41
#![allow(unused_assignments, unused_variables)]
52
// compile-flags: -C opt-level=3 # validates coverage now works with optimizations
63
extern crate used_crate;

‎tests/run-make/coverage/uses_inline_crate.rs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// FIXME #110395
2-
// ignore-llvm-cov-show-diffs
3-
41
#![allow(unused_assignments, unused_variables)]
52

63
// compile-flags: -C opt-level=3 # validates coverage now works with optimizations

0 commit comments

Comments
 (0)