Skip to content

Commit 6bacc1e

Browse files
hazefullyhazefully
committed
Merge branch 'main' into planner-phase-metrics
2 parents b53e042 + e816b7e commit 6bacc1e

File tree

322 files changed

+19258
-10993
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

322 files changed

+19258
-10993
lines changed

‎.github/workflows/nightly.yml‎

Lines changed: 125 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -6,39 +6,132 @@ on:
66
workflow_dispatch:
77

88
jobs:
9-
gradle:
9+
# In parallel, we run:
10+
# a) all the tests
11+
# b) the mixed-mode tests
12+
# c) the nightly tests
13+
# The nightly tests (currently) include all the regular tests, but we want to publish to teamscale the coverage
14+
# statistics we're getting from PRB separately. We want to be able to differentiate those two types of coverage
15+
# in teamscale for a better understanding of what might escape in PRB (or in our pre-release test run).
16+
# The mixed-mode coverage is a little different, we want to see explicitly what code is being covered in mixed-mode
17+
# so that we can find potential uncovered code that could expose bugs when multiple versions are running at the same
18+
# time.
19+
# It's possible that we could change the nightly tests to just be the new tests, and not the regular tests in the future.
20+
test:
1021
if: github.repository == 'FoundationDB/fdb-record-layer'
1122
runs-on: ubuntu-latest
23+
permissions:
24+
contents: read
1225
steps:
13-
- name: Checkout sources
14-
uses: actions/checkout@v4.2.2
15-
- name: Setup Base Environment
16-
uses: ./actions/setup-base-env
17-
- name: Setup FDB
18-
uses: ./actions/setup-fdb
19-
- name: Run Gradle Test
20-
uses: ./actions/gradle-test
21-
with:
22-
gradle_args: "-PreleaseBuild=false -PpublishBuild=false -PspotbugsEnableHtmlReport -Ptests.includeRandom -Ptests.iterations=2 -Ptests.nightly"
23-
- name: Publish Test Reports
24-
if: always()
25-
uses: actions/upload-artifact@v4.6.0
26-
with:
27-
name: test-reports
28-
path: |
29-
test-reports/fdb-java-annotations/
30-
test-reports/fdb-extensions/
31-
test-reports/fdb-record-layer-core/
32-
test-reports/fdb-record-layer-icu/
33-
test-reports/fdb-record-layer-spatial/
34-
test-reports/fdb-record-layer-lucene/
35-
test-reports/fdb-record-layer-jmh/
36-
test-reports/examples/
37-
test-reports/fdb-relational-api/
38-
test-reports/fdb-relational-core/
39-
test-reports/fdb-relational-cli/
40-
test-reports/fdb-relational-grpc/
41-
test-reports/fdb-relational-jdbc/
42-
test-reports/fdb-relational-server/
43-
test-reports/yaml-tests/
26+
- name: Checkout sources
27+
uses: actions/checkout@v4.2.2
28+
- name: Setup Base Environment
29+
id: setup-base
30+
uses: ./actions/setup-base-env
31+
- name: Setup FDB
32+
uses: ./actions/setup-fdb
33+
- name: Run Gradle Test
34+
uses: ./actions/gradle-test
35+
with:
36+
gradle_args: -PreleaseBuild=false -PpublishBuild=false
37+
- name: Upload coverage to teamscale
38+
# temporary until we validate that this is working correctly
39+
continue-on-error: true
40+
uses: ./actions/teamscale-upload
41+
with:
42+
partition: 'CI Tests'
43+
revision: ${{ github.sha }}
44+
files: "${{ github.workspace }}/.out/reports/jacoco/codeCoverageReport/codeCoverageReport.xml"
45+
teamscaleKey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}
46+
47+
mixed-mode-test:
48+
if: github.repository == 'FoundationDB/fdb-record-layer'
49+
runs-on: ubuntu-latest
50+
permissions:
51+
contents: read
52+
steps:
53+
- name: Checkout sources
54+
uses: actions/checkout@v4.2.2
55+
- name: Setup Base Environment
56+
uses: ./actions/setup-base-env
57+
- name: Setup FDB
58+
uses: ./actions/setup-fdb
59+
- name: Run Gradle Test
60+
uses: ./actions/gradle-test
61+
with:
62+
gradle_command: mixedModeTest
63+
gradle_args: -PreleaseBuild=false -PpublishBuild=false
64+
# We don't commit the incremented version, but we use this to know the version when generating
65+
# the resulting markdown
66+
- name: Increment version
67+
shell: bash
68+
run: python build/versionutils.py gradle.properties --increment -u PATCH
69+
- name: Get new version
70+
id: get_new_version
71+
shell: bash
72+
run: |
73+
echo "version=$(python build/versionutils.py gradle.properties)" >> "$GITHUB_OUTPUT"
74+
- name: Create markdown
75+
shell: bash
76+
run: python build/publish-mixed-mode-results.py ${{ steps.get_new_version.outputs.version }} --run-link ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} --output mixed-mode-results.md
77+
- name: Preview results
78+
shell: bash
79+
run: cat mixed-mode-results.md >> $GITHUB_STEP_SUMMARY
80+
- name: Upload coverage to teamscale
81+
# temporary until we validate that this is working correctly
82+
continue-on-error: true
83+
uses: ./actions/teamscale-upload
84+
with:
85+
partition: 'Mixed Mode Tests'
86+
revision: ${{ github.sha }}
87+
files: "${{ github.workspace }}/.out/reports/jacoco/codeCoverageReport/codeCoverageReport.xml"
88+
teamscaleKey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}
89+
90+
nightly_test:
91+
if: github.repository == 'FoundationDB/fdb-record-layer'
92+
runs-on: ubuntu-latest
93+
steps:
94+
- name: Checkout sources
95+
uses: actions/checkout@v4.2.2
96+
- name: Setup Base Environment
97+
uses: ./actions/setup-base-env
98+
- name: Setup FDB
99+
uses: ./actions/setup-fdb
100+
- name: Run Gradle Test
101+
uses: ./actions/gradle-test
102+
with:
103+
gradle_args: "-PreleaseBuild=false -PpublishBuild=false -PspotbugsEnableHtmlReport -Ptests.includeRandom -Ptests.iterations=2 -Ptests.nightly"
104+
- name: Publish Test Reports
105+
if: always()
106+
uses: actions/upload-artifact@v4.6.0
107+
with:
108+
name: test-reports
109+
path: |
110+
test-reports/fdb-java-annotations/
111+
test-reports/fdb-extensions/
112+
test-reports/fdb-record-layer-core/
113+
test-reports/fdb-record-layer-icu/
114+
test-reports/fdb-record-layer-spatial/
115+
test-reports/fdb-record-layer-lucene/
116+
test-reports/fdb-record-layer-jmh/
117+
test-reports/examples/
118+
test-reports/fdb-relational-api/
119+
test-reports/fdb-relational-core/
120+
test-reports/fdb-relational-cli/
121+
test-reports/fdb-relational-grpc/
122+
test-reports/fdb-relational-jdbc/
123+
test-reports/fdb-relational-server/
124+
test-reports/yaml-tests/
125+
126+
- name: Upload coverage to teamscale
127+
# temporary until we validate that this is working correctly
128+
continue-on-error: true
129+
uses: ./actions/teamscale-upload
130+
with:
131+
partition: 'Nightly Tests'
132+
revision: ${{ github.sha }}
133+
files: "${{ github.workspace }}/.out/reports/jacoco/codeCoverageReport/codeCoverageReport.xml"
134+
teamscaleKey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}
135+
136+
44137

‎.github/workflows/pull_request.yml‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
- name: Run Gradle Test
4747
uses: ./actions/gradle-test
4848
with:
49-
gradle_command: :fdb-record-layer-core:test :fdb-record-layer-core:destructiveTest
49+
gradle_command: :fdb-record-layer-core:jar :fdb-record-layer-core:test :fdb-record-layer-core:destructiveTest
5050
gradle_args: -PreleaseBuild=false -PpublishBuild=false
5151
- name: Publish Test Reports
5252
if: always()
@@ -61,6 +61,7 @@ jobs:
6161
name: core-coverage-data
6262
path: |
6363
**/.out/jacoco/*.exec
64+
**/.out/libs/*.jar
6465
include-hidden-files: true
6566
retention-days: 1
6667

@@ -80,7 +81,7 @@ jobs:
8081
- name: Run Gradle Test
8182
uses: ./actions/gradle-test
8283
with:
83-
gradle_command: :fdb-record-layer-lucene:test :fdb-record-layer-lucene:destructiveTest
84+
gradle_command: :fdb-record-layer-lucene:jar :fdb-record-layer-lucene:test :fdb-record-layer-lucene:destructiveTest
8485
gradle_args: -PreleaseBuild=false -PpublishBuild=false
8586
- name: Publish Test Reports
8687
if: always()
@@ -95,6 +96,7 @@ jobs:
9596
name: lucene-coverage-data
9697
path: |
9798
**/.out/jacoco/*.exec
99+
**/.out/libs/*.jar
98100
include-hidden-files: true
99101
retention-days: 1
100102

@@ -114,7 +116,13 @@ jobs:
114116
- name: Run Gradle Test
115117
uses: ./actions/gradle-test
116118
with:
117-
gradle_command: test -x :fdb-record-layer-core:test -x :fdb-record-layer-lucene:test destructiveTest -x :fdb-record-layer-core:destructiveTest -x :fdb-record-layer-lucene:destructiveTest
119+
gradle_command: >-
120+
test
121+
-x :fdb-record-layer-core:test
122+
-x :fdb-record-layer-lucene:test
123+
destructiveTest
124+
-x :fdb-record-layer-core:destructiveTest
125+
-x :fdb-record-layer-lucene:destructiveTest
118126
gradle_args: -PreleaseBuild=false -PpublishBuild=false
119127
- name: Publish Test Reports
120128
if: always()
@@ -156,12 +164,20 @@ jobs:
156164
uses: actions/checkout@v4.2.2
157165
- name: Setup Base Environment
158166
uses: ./actions/setup-base-env
159-
- name: 'Download artifacts'
167+
# It looks like, if you try to download them all as a pattern, the nested directories get stripped
168+
# so the coverage data (for e.g. lucene) does not end up in the appropirate subproject directory
169+
- name: 'Download lucene'
170+
uses: actions/download-artifact@v4
171+
with:
172+
name: lucene-coverage-data
173+
- name: 'Download core'
160174
uses: actions/download-artifact@v4
161175
with:
162-
pattern: |
163-
*-coverage-data
164-
merge-multiple: true
176+
name: core-coverage-data
177+
- name: 'Download other'
178+
uses: actions/download-artifact@v4
179+
with:
180+
name: other-coverage-data
165181
- name: Run JaCoCo Report
166182
uses: ./actions/run-gradle
167183
with:

‎.github/workflows/teamscale_upload.yml‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ name: Upload Coverage to Teamscale
33
permissions:
44
actions: read
55

6+
# This is a separate workflow from pull_request.yml because it needs to run with access
7+
# to the secrets
68
on:
79
workflow_run:
810
workflows: [Pull Request]
@@ -14,6 +16,10 @@ jobs:
1416
runs-on: ubuntu-latest
1517
if: ${{ github.event.workflow_run.conclusion == 'success' }}
1618
steps:
19+
# This checks out the latest commit on main, not teh PRs head
20+
# This is needed to get the teamscale-upload action
21+
- name: Checkout sources
22+
uses: actions/checkout@v4.2.2
1723
- name: 'Download artifact'
1824
uses: actions/github-script@v7
1925
with:
@@ -43,13 +49,9 @@ jobs:
4349
- name: 'Unzip artifact'
4450
run: unzip "${{ runner.temp }}/artifacts/coverage-report.zip" -d "${{ runner.temp }}/artifacts"
4551
- name: 'Upload coverage'
46-
uses: 'cqse/teamscale-upload-action@8d10b6693b9242420ef2062dbefc36af6e88d587'
52+
uses: ./actions/teamscale-upload
4753
with:
48-
server: 'https://fdb.teamscale.io'
49-
project: 'foundationdb-fdb-record-layer'
50-
user: 'fdb-record-layer-build'
5154
partition: 'CI Tests'
52-
accesskey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}
53-
format: 'JACOCO'
5455
revision: ${{ github.event.workflow_run.head_sha }}
5556
files: "${{ runner.temp}}/artifacts/codeCoverageReport.xml"
57+
teamscaleKey: ${{ secrets.TEAMSCALE_ACCESS_KEY }}

‎.teamscale.toml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Configuration for project foundationdb-fdb-record-layer, connector github-fdb-record-layer
2+
3+
# Learn more about the Teamscale IDE Configuration File Format at
4+
# <https://docs.teamscale.com/reference/teamscale-toml/>.
5+
6+
version = "1.0"
7+
8+
root = true
9+
10+
[server]
11+
url = "https://fdb.teamscale.io/"
12+
13+
[project]
14+
id = "foundationdb-fdb-record-layer"

‎actions/teamscale-upload/action.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Upload Teamscale Coverage
2+
3+
inputs:
4+
partition:
5+
description: 'Partition for the test coverage'
6+
required: true
7+
revision:
8+
description: 'Git sha'
9+
required: true
10+
files:
11+
description: 'Path to coverage xml file from jacoco'
12+
required: true
13+
teamscaleKey:
14+
description: 'Access key to teamscale'
15+
required: true
16+
17+
runs:
18+
using: "composite"
19+
steps:
20+
- name: Upload coverage to teamscale
21+
# temporary until we validate that this is working correctly
22+
continue-on-error: true
23+
uses: 'cqse/teamscale-upload-action@8d10b6693b9242420ef2062dbefc36af6e88d587'
24+
with:
25+
server: 'https://fdb.teamscale.io'
26+
project: 'foundationdb-fdb-record-layer'
27+
user: 'fdb-record-layer-build'
28+
partition: inputs.partition
29+
accesskey: ${{ inputs.teamscaleKey }}
30+
format: 'JACOCO'
31+
revision: ${{ inputs.revision }}
32+
files: ${{ inputs.files }}

0 commit comments

Comments
 (0)