Skip to content

Commit a1ad44c

Browse files
kevaundraykevaundray
committed
Merge branch 'unstable' into kw/sel-alternative
2 parents 64e5a45 + 90dd5bb commit a1ad44c

File tree

223 files changed

+5082
-4924
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

223 files changed

+5082
-4924
lines changed

‎.github/workflows/docker-reproducible.yml‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
name: docker-reproducible
2+
3+
on:
4+
push:
5+
branches:
6+
- unstable
7+
- stable
8+
tags:
9+
- v*
10+
workflow_dispatch: # allows manual triggering for testing purposes and skips publishing an image
11+
12+
env:
13+
DOCKER_REPRODUCIBLE_IMAGE_NAME: >-
14+
${{ github.repository_owner }}/lighthouse-reproducible
15+
DOCKER_PASSWORD: ${{ secrets.DH_KEY }}
16+
DOCKER_USERNAME: ${{ secrets.DH_ORG }}
17+
18+
jobs:
19+
extract-version:
20+
name: extract version
21+
runs-on: ubuntu-22.04
22+
steps:
23+
- name: Extract version
24+
run: |
25+
if [[ "${{ github.ref }}" == refs/tags/* ]]; then
26+
# It's a tag (e.g., v1.2.3)
27+
VERSION="${GITHUB_REF#refs/tags/}"
28+
elif [[ "${{ github.ref }}" == refs/heads/stable ]]; then
29+
# stable branch -> latest
30+
VERSION="latest"
31+
elif [[ "${{ github.ref }}" == refs/heads/unstable ]]; then
32+
# unstable branch -> latest-unstable
33+
VERSION="latest-unstable"
34+
else
35+
# For manual triggers from other branches and will not publish any image
36+
VERSION="test-build"
37+
fi
38+
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
39+
id: extract_version
40+
outputs:
41+
VERSION: ${{ steps.extract_version.outputs.VERSION }}
42+
43+
verify-and-build:
44+
name: verify reproducibility and build
45+
needs: extract-version
46+
strategy:
47+
matrix:
48+
arch: [amd64, arm64]
49+
include:
50+
- arch: amd64
51+
rust_target: x86_64-unknown-linux-gnu
52+
rust_image: >-
53+
rust:1.88-bullseye@sha256:8e3c421122bf4cd3b2a866af41a4dd52d87ad9e315fd2cb5100e87a7187a9816
54+
platform: linux/amd64
55+
runner: ubuntu-22.04
56+
- arch: arm64
57+
rust_target: aarch64-unknown-linux-gnu
58+
rust_image: >-
59+
rust:1.88-bullseye@sha256:8b22455a7ce2adb1355067638284ee99d21cc516fab63a96c4514beaf370aa94
60+
platform: linux/arm64
61+
runner: ubuntu-22.04-arm
62+
runs-on: ${{ matrix.runner }}
63+
steps:
64+
- uses: actions/checkout@v4
65+
66+
- name: Set up Docker Buildx
67+
uses: docker/setup-buildx-action@v3
68+
with:
69+
driver: docker
70+
71+
- name: Verify reproducible builds (${{ matrix.arch }})
72+
run: |
73+
# Build first image
74+
docker build -f Dockerfile.reproducible \
75+
--platform ${{ matrix.platform }} \
76+
--build-arg RUST_TARGET="${{ matrix.rust_target }}" \
77+
--build-arg RUST_IMAGE="${{ matrix.rust_image }}" \
78+
-t lighthouse-verify-1-${{ matrix.arch }} .
79+
80+
# Extract binary from first build
81+
docker create --name extract-1-${{ matrix.arch }} lighthouse-verify-1-${{ matrix.arch }}
82+
docker cp extract-1-${{ matrix.arch }}:/lighthouse ./lighthouse-1-${{ matrix.arch }}
83+
docker rm extract-1-${{ matrix.arch }}
84+
85+
# Clean state for second build
86+
docker buildx prune -f
87+
docker system prune -f
88+
89+
# Build second image
90+
docker build -f Dockerfile.reproducible \
91+
--platform ${{ matrix.platform }} \
92+
--build-arg RUST_TARGET="${{ matrix.rust_target }}" \
93+
--build-arg RUST_IMAGE="${{ matrix.rust_image }}" \
94+
-t lighthouse-verify-2-${{ matrix.arch }} .
95+
96+
# Extract binary from second build
97+
docker create --name extract-2-${{ matrix.arch }} lighthouse-verify-2-${{ matrix.arch }}
98+
docker cp extract-2-${{ matrix.arch }}:/lighthouse ./lighthouse-2-${{ matrix.arch }}
99+
docker rm extract-2-${{ matrix.arch }}
100+
101+
# Compare binaries
102+
echo "=== Comparing binaries ==="
103+
echo "Build 1 SHA256: $(sha256sum lighthouse-1-${{ matrix.arch }})"
104+
echo "Build 2 SHA256: $(sha256sum lighthouse-2-${{ matrix.arch }})"
105+
106+
if cmp lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }}; then
107+
echo "Reproducible build verified for ${{ matrix.arch }}"
108+
else
109+
echo "Reproducible build FAILED for ${{ matrix.arch }}"
110+
echo "BLOCKING RELEASE: Builds are not reproducible!"
111+
echo "First 10 differences:"
112+
cmp -l lighthouse-1-${{ matrix.arch }} lighthouse-2-${{ matrix.arch }} | head -10
113+
exit 1
114+
fi
115+
116+
# Clean up verification artifacts but keep one image for publishing
117+
rm -f lighthouse-*-${{ matrix.arch }}
118+
docker rmi lighthouse-verify-1-${{ matrix.arch }} || true
119+
120+
# Re-tag the second image for publishing (we verified it's identical to first)
121+
VERSION=${{ needs.extract-version.outputs.VERSION }}
122+
FINAL_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
123+
docker tag lighthouse-verify-2-${{ matrix.arch }} "$FINAL_TAG"
124+
125+
- name: Log in to Docker Hub
126+
if: ${{ github.event_name != 'workflow_dispatch' }}
127+
uses: docker/login-action@v3
128+
with:
129+
username: ${{ env.DOCKER_USERNAME }}
130+
password: ${{ env.DOCKER_PASSWORD }}
131+
132+
- name: Push verified image (${{ matrix.arch }})
133+
if: ${{ github.event_name != 'workflow_dispatch' }}
134+
run: |
135+
VERSION=${{ needs.extract-version.outputs.VERSION }}
136+
IMAGE_TAG="${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}"
137+
docker push "$IMAGE_TAG"
138+
139+
- name: Clean up local images
140+
run: |
141+
docker rmi lighthouse-verify-2-${{ matrix.arch }} || true
142+
VERSION=${{ needs.extract-version.outputs.VERSION }}
143+
docker rmi "${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${VERSION}-${{ matrix.arch }}" || true
144+
145+
- name: Upload verification artifacts (on failure)
146+
if: failure()
147+
uses: actions/upload-artifact@v4
148+
with:
149+
name: verification-failure-${{ matrix.arch }}
150+
path: |
151+
lighthouse-*-${{ matrix.arch }}
152+
153+
create-manifest:
154+
name: create multi-arch manifest
155+
runs-on: ubuntu-22.04
156+
needs: [extract-version, verify-and-build]
157+
if: ${{ github.event_name != 'workflow_dispatch' }}
158+
steps:
159+
- name: Log in to Docker Hub
160+
uses: docker/login-action@v3
161+
with:
162+
username: ${{ env.DOCKER_USERNAME }}
163+
password: ${{ env.DOCKER_PASSWORD }}
164+
165+
- name: Create and push multi-arch manifest
166+
run: |
167+
IMAGE_NAME=${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}
168+
VERSION=${{ needs.extract-version.outputs.VERSION }}
169+
170+
# Create manifest for the version tag
171+
docker manifest create \
172+
${IMAGE_NAME}:${VERSION} \
173+
${IMAGE_NAME}:${VERSION}-amd64 \
174+
${IMAGE_NAME}:${VERSION}-arm64
175+
176+
docker manifest push ${IMAGE_NAME}:${VERSION}

‎.github/workflows/local-testnet.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ concurrency:
1414

1515
jobs:
1616
dockerfile-ubuntu:
17-
runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest' }}
17+
runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
1818
steps:
1919
- uses: actions/checkout@v5
2020

@@ -31,7 +31,7 @@ jobs:
3131
retention-days: 3
3232

3333
run-local-testnet:
34-
runs-on: ubuntu-22.04
34+
runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
3535
needs: dockerfile-ubuntu
3636
steps:
3737
- uses: actions/checkout@v5
@@ -89,7 +89,7 @@ jobs:
8989
${{ steps.assertoor_test_result.outputs.failed_test_details }}
9090
EOF
9191
)
92-
92+
9393
echo "Test Result: $test_result"
9494
echo "$test_status"
9595
if ! [ "$test_result" == "success" ]; then
@@ -100,7 +100,7 @@ jobs:
100100
101101
doppelganger-protection-success-test:
102102
needs: dockerfile-ubuntu
103-
runs-on: ubuntu-22.04
103+
runs-on: ubuntu-latest
104104
steps:
105105
- uses: actions/checkout@v5
106106

@@ -136,7 +136,7 @@ jobs:
136136

137137
doppelganger-protection-failure-test:
138138
needs: dockerfile-ubuntu
139-
runs-on: ubuntu-22.04
139+
runs-on: ubuntu-latest
140140
steps:
141141
- uses: actions/checkout@v5
142142

@@ -173,7 +173,7 @@ jobs:
173173
# Tests checkpoint syncing to a live network (current fork) and a running devnet (usually next scheduled fork)
174174
checkpoint-sync-test:
175175
name: checkpoint-sync-test-${{ matrix.network }}
176-
runs-on: ubuntu-latest
176+
runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
177177
needs: dockerfile-ubuntu
178178
if: contains(github.event.pull_request.labels.*.name, 'syncing')
179179
continue-on-error: true
@@ -216,7 +216,7 @@ jobs:
216216
# Test syncing from genesis on a local testnet. Aims to cover forward syncing both short and long distances.
217217
genesis-sync-test:
218218
name: genesis-sync-test-${{ matrix.fork }}-${{ matrix.offline_secs }}s
219-
runs-on: ubuntu-latest
219+
runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
220220
needs: dockerfile-ubuntu
221221
strategy:
222222
matrix:
@@ -259,7 +259,7 @@ jobs:
259259
# a PR is safe to merge. New jobs should be added here.
260260
local-testnet-success:
261261
name: local-testnet-success
262-
runs-on: ubuntu-latest
262+
runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
263263
needs: [
264264
'dockerfile-ubuntu',
265265
'run-local-testnet',
@@ -272,4 +272,4 @@ jobs:
272272
- name: Check that success job is dependent on all others
273273
run: |
274274
exclude_jobs='checkpoint-sync-test'
275-
./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success "$exclude_jobs"
275+
./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success "$exclude_jobs"

‎.github/workflows/nightly-tests.yml‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
# We only run tests on `RECENT_FORKS` on CI. To make sure we don't break prior forks, we run nightly tests to cover all prior forks.
2+
name: nightly-tests
3+
4+
on:
5+
schedule:
6+
# Run at 8:30 AM UTC every day
7+
- cron: '30 8 * * *'
8+
workflow_dispatch: # Allow manual triggering
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
env:
15+
# Deny warnings in CI
16+
# Disable debug info (see https://github.com/sigp/lighthouse/issues/4005)
17+
RUSTFLAGS: "-D warnings -C debuginfo=0"
18+
# Prevent Github API rate limiting.
19+
LIGHTHOUSE_GITHUB_TOKEN: ${{ secrets.LIGHTHOUSE_GITHUB_TOKEN }}
20+
# Disable incremental compilation
21+
CARGO_INCREMENTAL: 0
22+
# Enable portable to prevent issues with caching `blst` for the wrong CPU type
23+
TEST_FEATURES: portable
24+
25+
jobs:
26+
setup-matrix:
27+
name: setup-matrix
28+
runs-on: ubuntu-latest
29+
outputs:
30+
forks: ${{ steps.set-matrix.outputs.forks }}
31+
steps:
32+
- name: Set matrix
33+
id: set-matrix
34+
run: |
35+
# All prior forks to cover in nightly tests. This list should be updated when we remove a fork from `RECENT_FORKS`.
36+
echo 'forks=["phase0", "altair", "bellatrix", "capella", "deneb"]' >> $GITHUB_OUTPUT
37+
38+
beacon-chain-tests:
39+
name: beacon-chain-tests
40+
needs: setup-matrix
41+
runs-on: 'ubuntu-latest'
42+
env:
43+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
44+
strategy:
45+
matrix:
46+
fork: ${{ fromJson(needs.setup-matrix.outputs.forks) }}
47+
fail-fast: false
48+
steps:
49+
- uses: actions/checkout@v5
50+
- name: Get latest version of stable Rust
51+
uses: moonrepo/setup-rust@v1
52+
with:
53+
channel: stable
54+
cache-target: release
55+
bins: cargo-nextest
56+
- name: Run beacon_chain tests for ${{ matrix.fork }}
57+
run: make test-beacon-chain-${{ matrix.fork }}
58+
timeout-minutes: 60
59+
60+
http-api-tests:
61+
name: http-api-tests
62+
needs: setup-matrix
63+
runs-on: 'ubuntu-latest'
64+
env:
65+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
66+
strategy:
67+
matrix:
68+
fork: ${{ fromJson(needs.setup-matrix.outputs.forks) }}
69+
fail-fast: false
70+
steps:
71+
- uses: actions/checkout@v5
72+
- name: Get latest version of stable Rust
73+
uses: moonrepo/setup-rust@v1
74+
with:
75+
channel: stable
76+
cache-target: release
77+
bins: cargo-nextest
78+
- name: Run http_api tests for ${{ matrix.fork }}
79+
run: make test-http-api-${{ matrix.fork }}
80+
timeout-minutes: 60
81+
82+
op-pool-tests:
83+
name: op-pool-tests
84+
needs: setup-matrix
85+
runs-on: ubuntu-latest
86+
env:
87+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
88+
strategy:
89+
matrix:
90+
fork: ${{ fromJson(needs.setup-matrix.outputs.forks) }}
91+
fail-fast: false
92+
steps:
93+
- uses: actions/checkout@v5
94+
- name: Get latest version of stable Rust
95+
uses: moonrepo/setup-rust@v1
96+
with:
97+
channel: stable
98+
cache-target: release
99+
bins: cargo-nextest
100+
- name: Run operation_pool tests for ${{ matrix.fork }}
101+
run: make test-op-pool-${{ matrix.fork }}
102+
timeout-minutes: 60
103+
104+
network-tests:
105+
name: network-tests
106+
needs: setup-matrix
107+
runs-on: ubuntu-latest
108+
env:
109+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
110+
strategy:
111+
matrix:
112+
fork: ${{ fromJson(needs.setup-matrix.outputs.forks) }}
113+
fail-fast: false
114+
steps:
115+
- uses: actions/checkout@v5
116+
- name: Get latest version of stable Rust
117+
uses: moonrepo/setup-rust@v1
118+
with:
119+
channel: stable
120+
cache-target: release
121+
bins: cargo-nextest
122+
- name: Create CI logger dir
123+
run: mkdir ${{ runner.temp }}/network_test_logs
124+
- name: Run network tests for ${{ matrix.fork }}
125+
run: make test-network-${{ matrix.fork }}
126+
timeout-minutes: 60
127+
env:
128+
TEST_FEATURES: portable
129+
CI_LOGGER_DIR: ${{ runner.temp }}/network_test_logs
130+
- name: Upload logs
131+
if: always()
132+
uses: actions/upload-artifact@v4
133+
with:
134+
name: network_test_logs_${{ matrix.fork }}
135+
path: ${{ runner.temp }}/network_test_logs

0 commit comments

Comments
 (0)