Skip to content

Commit cd41f9a

Browse files
ScottDugasScottDugas
authored
Upload coverage from nightly builds (#3568)
Add new jobs to the nightly flow to run standard tests and mixed-mode tests. This is primarily there so that we can upload coverage to teamscale. These new teamscale steps are set to continue-on-error, to make sure they are working correctly before impacting the nightly build. I also extracted the teamscale upload into a separate action to decrease repetition, this is the biggest risk; if I messed this up, it could impact the post-pr teamscale upload.
1 parent e409db3 commit cd41f9a

File tree

3 files changed

+161
-38
lines changed

3 files changed

+161
-38
lines changed

‎.github/workflows/nightly.yml‎

Lines changed: 125 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -6,39 +6,132 @@ on:
66
workflow_dispatch:
77

88
jobs:
9-
gradle:
9+
# In parallel, we run:
10+
# a) all the tests
11+
# b) the mixed-mode tests
12+
# c) the nightly tests
13+
# The nightly tests (currently) include all the regular tests, but we want to publish to teamscale the coverage
14+
# statistics we're getting from PRB separately. We want to be able to differentiate those two types of coverage
15+
# in teamscale for a better understanding of what might escape in PRB (or in our pre-release test run).
16+
# The mixed-mode coverage is a little different, we want to see explicitly what code is being covered in mixed-mode
17+
# so that we can find potential uncovered code that could expose bugs when multiple versions are running at the same
18+
# time.
19+
# It's possible that we could change the nightly tests to just be the new tests, and not the regular tests in the future.
20+
test:
1021
if: github.repository == 'FoundationDB/fdb-record-layer'
1122
runs-on: ubuntu-latest
23+
permissions:
24+
contents: read
1225
steps:
13-
- name: Checkout sources
14-
uses: actions/checkout@v4.2.2
15-
- name: Setup Base Environment
16-
uses: ./actions/setup-base-env
17-
- name: Setup FDB
18-
uses: ./actions/setup-fdb
19-
- name: Run Gradle Test
20-
uses: ./actions/gradle-test
21-
with:
22-
gradle_args: "-PreleaseBuild=false -PpublishBuild=false -PspotbugsEnableHtmlReport -Ptests.includeRandom -Ptests.iterations=2 -Ptests.nightly"
23-
- name: Publish Test Reports
24-
if: always()
25-
uses: actions/upload-artifact@v4.6.0
26-
with:
27-
name: test-reports
28-
path: |
29-
test-reports/fdb-java-annotations/
30-
test-reports/fdb-extensions/
31-
test-reports/fdb-record-layer-core/
32-
test-reports/fdb-record-layer-icu/
33-
test-reports/fdb-record-layer-spatial/
34-
test-reports/fdb-record-layer-lucene/
35-
test-reports/fdb-record-layer-jmh/
36-
test-reports/examples/
37-
test-reports/fdb-relational-api/
38-
test-reports/fdb-relational-core/
39-
test-reports/fdb-relational-cli/
40-
test-reports/fdb-relational-grpc/
41-
test-reports/fdb-relational-jdbc/
42-
test-reports/fdb-relational-server/
43-
test-reports/yaml-tests/
26+
- name: Checkout sources
27+
uses: actions/checkout@v4.2.2
28+
- name: Setup Base Environment
29+
id: setup-base
30+
uses: ./actions/setup-base-env
31+
- name: Setup FDB
32+
uses: ./actions/setup-fdb
33+
- name: Run Gradle Test
34+
uses: ./actions/gradle-test
35+
with:
36+
gradle_args: -PreleaseBuild=true -PpublishBuild=true
37+
- name: Upload coverage to teamscale
38+
# temporary until we validate that this is working correctly
39+
continue-on-error: true
40+
uses: ./actions/teamscale-upload
41+
with:
42+
partition: 'CI Tests'
43+
revision: ${{ github.sha }}
44+
files: "${{ github.workspace }}/.out/reports/jacoco/codeCoverageReport/codeCoverageReport.xml"
45+
teamscaleKey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}
46+
47+
mixed-mode-test:
48+
if: github.repository == 'FoundationDB/fdb-record-layer'
49+
runs-on: ubuntu-latest
50+
permissions:
51+
contents: read
52+
steps:
53+
- name: Checkout sources
54+
uses: actions/checkout@v4.2.2
55+
- name: Setup Base Environment
56+
uses: ./actions/setup-base-env
57+
- name: Setup FDB
58+
uses: ./actions/setup-fdb
59+
- name: Run Gradle Test
60+
uses: ./actions/gradle-test
61+
with:
62+
gradle_command: mixedModeTest
63+
gradle_args: -PreleaseBuild=false -PpublishBuild=false
64+
# We don't commit the incremented version, but we use this to know the version when generating
65+
# the resulting markdown
66+
- name: Increment version
67+
shell: bash
68+
run: python build/versionutils.py gradle.properties --increment -u PATCH
69+
- name: Get new version
70+
id: get_new_version
71+
shell: bash
72+
run: |
73+
echo "version=$(python build/versionutils.py gradle.properties)" >> "$GITHUB_OUTPUT"
74+
- name: Create markdown
75+
shell: bash
76+
run: python build/publish-mixed-mode-results.py ${{ steps.get_new_version.outputs.version }} --run-link ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} --output mixed-mode-results.md
77+
- name: Preview results
78+
shell: bash
79+
run: cat mixed-mode-results.md >> $GITHUB_STEP_SUMMARY
80+
- name: Upload coverage to teamscale
81+
# temporary until we validate that this is working correctly
82+
continue-on-error: true
83+
uses: ./actions/teamscale-upload
84+
with:
85+
partition: 'Mixed Mode Tests'
86+
revision: ${{ github.sha }}
87+
files: "${{ github.workspace }}/.out/reports/jacoco/codeCoverageReport/codeCoverageReport.xml"
88+
teamscaleKey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}
89+
90+
nightly_test:
91+
if: github.repository == 'FoundationDB/fdb-record-layer'
92+
runs-on: ubuntu-latest
93+
steps:
94+
- name: Checkout sources
95+
uses: actions/checkout@v4.2.2
96+
- name: Setup Base Environment
97+
uses: ./actions/setup-base-env
98+
- name: Setup FDB
99+
uses: ./actions/setup-fdb
100+
- name: Run Gradle Test
101+
uses: ./actions/gradle-test
102+
with:
103+
gradle_args: "-PreleaseBuild=false -PpublishBuild=false -PspotbugsEnableHtmlReport -Ptests.includeRandom -Ptests.iterations=2 -Ptests.nightly"
104+
- name: Publish Test Reports
105+
if: always()
106+
uses: actions/upload-artifact@v4.6.0
107+
with:
108+
name: test-reports
109+
path: |
110+
test-reports/fdb-java-annotations/
111+
test-reports/fdb-extensions/
112+
test-reports/fdb-record-layer-core/
113+
test-reports/fdb-record-layer-icu/
114+
test-reports/fdb-record-layer-spatial/
115+
test-reports/fdb-record-layer-lucene/
116+
test-reports/fdb-record-layer-jmh/
117+
test-reports/examples/
118+
test-reports/fdb-relational-api/
119+
test-reports/fdb-relational-core/
120+
test-reports/fdb-relational-cli/
121+
test-reports/fdb-relational-grpc/
122+
test-reports/fdb-relational-jdbc/
123+
test-reports/fdb-relational-server/
124+
test-reports/yaml-tests/
125+
126+
- name: Upload coverage to teamscale
127+
# temporary until we validate that this is working correctly
128+
continue-on-error: true
129+
uses: ./actions/teamscale-upload
130+
with:
131+
partition: 'Nightly Tests'
132+
revision: ${{ github.sha }}
133+
files: "${{ github.workspace }}/.out/reports/jacoco/codeCoverageReport/codeCoverageReport.xml"
134+
teamscaleKey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}
135+
136+
44137

‎.github/workflows/teamscale_upload.yml‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ name: Upload Coverage to Teamscale
33
permissions:
44
actions: read
55

6+
# This is a separate workflow from pull_request.yml because it needs to run with access
7+
# to the secrets
68
on:
79
workflow_run:
810
workflows: [Pull Request]
@@ -43,13 +45,9 @@ jobs:
4345
- name: 'Unzip artifact'
4446
run: unzip "${{ runner.temp }}/artifacts/coverage-report.zip" -d "${{ runner.temp }}/artifacts"
4547
- name: 'Upload coverage'
46-
uses: 'cqse/teamscale-upload-action@8d10b6693b9242420ef2062dbefc36af6e88d587'
48+
uses: ./actions/teamscale-upload
4749
with:
48-
server: 'https://fdb.teamscale.io'
49-
project: 'foundationdb-fdb-record-layer'
50-
user: 'fdb-record-layer-build'
5150
partition: 'CI Tests'
52-
accesskey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}
53-
format: 'JACOCO'
5451
revision: ${{ github.event.workflow_run.head_sha }}
5552
files: "${{ runner.temp}}/artifacts/codeCoverageReport.xml"
53+
teamscaleKey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}

‎actions/teamscale-upload/action.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Upload Teamscale Coverage
2+
3+
inputs:
4+
partition:
5+
description: 'Partition for the test coverage'
6+
required: true
7+
revision:
8+
description: 'Git sha'
9+
required: true
10+
files:
11+
description: 'Path to coverage xml file from jacoco'
12+
required: true
13+
teamscaleKey:
14+
description: 'Access key to teamscale'
15+
required: true
16+
17+
runs:
18+
using: "composite"
19+
steps:
20+
- name: Upload coverage to teamscale
21+
# temporary until we validate that this is working correctly
22+
continue-on-error: true
23+
uses: 'cqse/teamscale-upload-action@8d10b6693b9242420ef2062dbefc36af6e88d587'
24+
with:
25+
server: 'https://fdb.teamscale.io'
26+
project: 'foundationdb-fdb-record-layer'
27+
user: 'fdb-record-layer-build'
28+
partition: inputs.partition
29+
accesskey: ${{ inputs.teamscaleKey }}
30+
format: 'JACOCO'
31+
revision: ${{ inputs.revision }}
32+
files: ${{ inputs.files }}

0 commit comments

Comments
 (0)