Skip to content

Commit f3e8ceb

Browse files
maennchenmaennchen
authored
Implement Test Coverage Reporting (#14343)
1 parent 32571b4 commit f3e8ceb

File tree

18 files changed

+211
-14
lines changed

18 files changed

+211
-14
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,16 @@ permissions:
2121

2222
jobs:
2323
test_linux:
24-
name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }}
24+
name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }}${{ matrix.coverage && ' (coverage)' || '' }}
2525
strategy:
2626
fail-fast: false
2727
matrix:
2828
include:
2929
- otp_version: "27.1"
3030
deterministic: true
31+
- otp_version: "27.1"
32+
erlc_opts: "warnings_as_errors"
33+
coverage: true
3134
- otp_version: "27.1"
3235
otp_latest: true
3336
erlc_opts: "warnings_as_errors"
@@ -68,6 +71,11 @@ jobs:
6871
- name: Elixir test suite
6972
run: make test_elixir
7073
continue-on-error: ${{ matrix.development }}
74+
env:
75+
COVER: "${{ matrix.coverage }}"
76+
- name: "Calculate Coverage"
77+
run: make cover | tee "$GITHUB_STEP_SUMMARY"
78+
if: "${{ matrix.coverage }}"
7179
- name: Build docs (ExDoc main)
7280
if: ${{ matrix.otp_latest }}
7381
run: |
@@ -85,6 +93,12 @@ jobs:
8593
# Recompile System without .git
8694
cd lib/elixir && ../../bin/elixirc -o ebin lib/system.ex && cd -
8795
taskset 1 make check_reproducible
96+
- name: "Upload Coverage Artifact"
97+
if: "${{ matrix.coverage }}"
98+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
99+
with:
100+
name: TestCoverage
101+
path: cover/*
88102

89103
test_windows:
90104
name: Windows Server 2019, Erlang/OTP ${{ matrix.otp_version }}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,4 @@
1515
/.eunit
1616
.elixir.plt
1717
erl_crash.dump
18+
/cover/

‎Makefile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ lib/$(1)/ebin/Elixir.$(2).beam: $(wildcard lib/$(1)/lib/*.ex) $(wildcard lib/$(1
5353
test_$(1): test_formatted $(1)
5454
@ echo "==> $(1) (ex_unit)"
5555
$(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/$(TEST_FILES)";
56+
57+
cover/ex_unit_$(1).coverdata:
58+
$(Q) COVER="1" $(MAKE) test_$(1)
59+
cover/combined.coverdata: cover/ex_unit_$(1).coverdata
5660
endef
5761

5862
define WRITE_SOURCE_DATE_EPOCH
@@ -175,6 +179,7 @@ clean: clean_man
175179
rm -rf lib/mix/test/fixtures/git_sparse_repo/
176180
rm -rf lib/mix/test/fixtures/archive/ebin/
177181
rm -f erl_crash.dump
182+
rm -rf cover
178183

179184
clean_elixir:
180185
$(Q) rm -f lib/*/ebin/Elixir.*.beam
@@ -287,6 +292,16 @@ test_stdlib: compile
287292
cd lib/elixir && ../../bin/elixir --sname primary -r "test/elixir/test_helper.exs" -pr "test/elixir/**/$(TEST_FILES)"; \
288293
fi
289294

295+
cover/ex_unit_elixir.coverdata:
296+
$(Q) COVER="1" $(MAKE) test_stdlib
297+
cover/combined.coverdata: cover/ex_unit_elixir.coverdata
298+
299+
cover/combined.coverdata:
300+
bin/elixir ./lib/elixir/scripts/cover.exs
301+
302+
.PHONY: cover
303+
cover: cover/combined.coverdata
304+
290305
#==> Dialyzer tasks
291306

292307
DIALYZER_OPTS = --no_check_plt --fullpath -Werror_handling -Wunmatched_returns -Wunderspecs

‎lib/eex/test/test_helper.exs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
{line_exclude, line_include} =
66
if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []}
77

8+
Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__)
9+
CoverageRecorder.maybe_record("eex")
10+
811
ExUnit.start(
912
trace: !!System.get_env("TRACE"),
1013
include: line_include,

‎lib/elixir/scripts/cover.exs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!bin/elixir
2+
3+
# SPDX-License-Identifier: Apache-2.0
4+
# SPDX-FileCopyrightText: 2021 The Elixir Team
5+
6+
Code.require_file("cover_record.exs", __DIR__)
7+
cover_pid = CoverageRecorder.enable_coverage()
8+
9+
coverdata_inputs =
10+
CoverageRecorder.cover_dir() |> Path.join("ex_unit_*.coverdata") |> Path.wildcard()
11+
12+
coverdata_output = Path.join(CoverageRecorder.cover_dir(), "combined.coverdata")
13+
14+
for file <- coverdata_inputs do
15+
:ok = :cover.import(String.to_charlist(file))
16+
end
17+
18+
:ok = :cover.export(String.to_charlist(coverdata_output))
19+
20+
{:ok, _} = Application.ensure_all_started(:mix)
21+
22+
# Silence analyse import messages emitted by cover
23+
{:ok, string_io} = StringIO.open("")
24+
Process.group_leader(cover_pid, string_io)
25+
26+
:ok =
27+
Mix.Tasks.Test.Coverage.generate_cover_results(
28+
output: CoverageRecorder.cover_dir(),
29+
summary: [threshold: 0]
30+
)

‎lib/elixir/scripts/cover_record.exs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
# SPDX-FileCopyrightText: 2021 The Elixir Team
3+
4+
defmodule CoverageRecorder do
5+
def maybe_record(suite_name) do
6+
if enabled?() do
7+
record(suite_name)
8+
9+
true
10+
else
11+
false
12+
end
13+
end
14+
15+
def enable_coverage do
16+
_ = :cover.stop()
17+
{:ok, pid} = :cover.start()
18+
19+
cover_compile_ebins()
20+
21+
pid
22+
end
23+
24+
def cover_dir, do: Path.join(root_dir(), "cover")
25+
26+
defp enabled? do
27+
case System.fetch_env("COVER") do
28+
{:ok, truthy} when truthy in ~w[1 true yes y] ->
29+
true
30+
31+
_ ->
32+
false
33+
end
34+
end
35+
36+
defp root_dir, do: Path.join(__DIR__, "../../..")
37+
defp ebins, do: root_dir() |> Path.join("lib/*/ebin") |> Path.wildcard()
38+
39+
defp record(suite_name) do
40+
file = Path.join(cover_dir(), "ex_unit_#{suite_name}.coverdata")
41+
42+
enable_coverage()
43+
44+
System.at_exit(fn _status ->
45+
File.mkdir_p!(cover_dir())
46+
47+
:ok = :cover.export(String.to_charlist(file))
48+
end)
49+
end
50+
51+
defp cover_compile_ebins do
52+
relevant_beam_files()
53+
|> Enum.map(&String.to_charlist/1)
54+
|> :cover.compile_beam()
55+
|> Enum.each(fn
56+
{:ok, _module} ->
57+
:ok
58+
59+
{:error, reason} ->
60+
raise "Failed to cover compile with reason: #{inspect(reason)}"
61+
end)
62+
end
63+
64+
defp relevant_beam_files do
65+
ebins()
66+
|> Enum.flat_map(fn ebin ->
67+
ebin |> Path.join("*.beam") |> Path.wildcard()
68+
end)
69+
|> Enum.reject(&deprecated/1)
70+
end
71+
72+
defp deprecated(file) do
73+
mod = file |> Path.basename(".beam") |> String.to_atom()
74+
75+
match?({:docs_v1, _, _, _, _, %{deprecated: _}, _}, Code.fetch_docs(mod))
76+
end
77+
end

‎lib/elixir/test/elixir/exception_test.exs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -888,6 +888,7 @@ defmodule ExceptionTest do
888888
assert stack == [{BlameModule, :fun, 1, [line: 13]}]
889889
end
890890

891+
@tag :require_ast
891892
test "annotates args and clauses from mfa" do
892893
import PathHelpers
893894

‎lib/elixir/test/elixir/kernel/dialyzer_test.exs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ defmodule Kernel.DialyzerTest do
88
use ExUnit.Case, async: true
99

1010
@moduletag :dialyzer
11+
@moduletag :require_ast
1112
import PathHelpers
1213

1314
setup_all do

‎lib/elixir/test/elixir/module/types/integration_test.exs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,7 @@ defmodule Module.Types.IntegrationTest do
458458
assert_warnings(files, warnings)
459459
end
460460

461+
@tag :require_ast
461462
test "String.Chars protocol dispatch" do
462463
files = %{
463464
"a.ex" => """
@@ -520,6 +521,7 @@ defmodule Module.Types.IntegrationTest do
520521
assert_warnings(files, warnings, consolidate_protocols: true)
521522
end
522523

524+
@tag :require_ast
523525
test "Enumerable protocol dispatch" do
524526
files = %{
525527
"a.ex" => """

‎lib/elixir/test/elixir/test_helper.exs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,20 @@ source_exclude =
124124
[]
125125
end
126126

127+
Code.require_file("../../scripts/cover_record.exs", __DIR__)
128+
129+
cover_exclude =
130+
if CoverageRecorder.maybe_record("elixir") do
131+
[:require_ast]
132+
else
133+
[]
134+
end
135+
127136
ExUnit.start(
128137
trace: !!System.get_env("TRACE"),
129138
assert_receive_timeout: assert_timeout,
130-
exclude: epmd_exclude ++ os_exclude ++ line_exclude ++ distributed_exclude ++ source_exclude,
139+
exclude:
140+
epmd_exclude ++
141+
os_exclude ++ line_exclude ++ distributed_exclude ++ source_exclude ++ cover_exclude,
131142
include: line_include
132143
)

0 commit comments

Comments
 (0)