Skip to content

Commit 8a121b6

Browse files
ayush-pantaayush-panta
authored
ci: add samcli tests through lima vm on codebuild (runfinch#331)
This is an implementation of automated tests for SAMcli against Finch. It runs in a CodeBuild Mac runner so that it may start the Lima VM. Due to issues in the CodeBuild environment this test suite must run in sequence, taking about 1 hour 45 minutes to complete. This will therefore be run nightly, with the option for manual trigger if needed. --------- Signed-off-by: ayush-panta <ayushkp@amazon.com>
1 parent b91e941 commit 8a121b6

File tree

8 files changed

+535
-0
lines changed

8 files changed

+535
-0
lines changed

‎.github/workflows/samcli-vm.yaml‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
name: samcli-vm
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
schedule:
8+
- cron: '0 8 * * *'
9+
workflow_dispatch:
10+
11+
env:
12+
GO_VERSION: '1.24.x'
13+
PYTHON_VERSION: '3.11'
14+
PYTHON_BINARY: 'python3.11'
15+
AWS_DEFAULT_REGION: "${{ secrets.REGION }}"
16+
BY_CANARY: true # allows full testing
17+
SAM_CLI_DEV: 1
18+
SAM_CLI_TELEMETRY: 0
19+
DOCKER_HOST: unix:///Applications/Finch/lima/data/finch/sock/finch.sock
20+
DOCKER_CONFIG: /Users/ec2-user/.finch
21+
22+
permissions:
23+
id-token: write
24+
contents: read
25+
26+
jobs:
27+
samcli-vm-test:
28+
runs-on: codebuild-finch-daemon-arm64-2-instance-${{ github.run_id }}-${{ github.run_attempt }}
29+
steps:
30+
31+
- name: Clean macOS runner workspace
32+
run: |
33+
rm -rf ${{ github.workspace }}/*
34+
35+
- name: Configure Git for ec2-user
36+
run: |
37+
git config --global --add safe.directory "*"
38+
shell: bash
39+
40+
- name: Set up Go
41+
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
42+
with:
43+
go-version: ${{ env.GO_VERSION }}
44+
cache: false
45+
46+
- name: Configure Go for ec2-user
47+
run: |
48+
chown -R ec2-user:staff $GOPATH || true
49+
chown -R ec2-user:staff $RUNNER_TOOL_CACHE/go || true
50+
51+
- name: Install Rosetta 2
52+
run: su ec2-user -c 'echo "A" | /usr/sbin/softwareupdate --install-rosetta --agree-to-license || true'
53+
54+
- name: Set up Python
55+
uses: actions/setup-python@v5
56+
with:
57+
python-version: ${{ env.PYTHON_VERSION }}
58+
59+
- name: Configure Python for ec2-user
60+
run: |
61+
# Make Python accessible to ec2-user
62+
chown -R ec2-user:staff $(${{ env.PYTHON_BINARY }} -c "import site; print(site.USER_BASE)") || true
63+
# Or symlink to ec2-user's PATH
64+
ln -sf $(which ${{ env.PYTHON_BINARY }}) /usr/local/bin/${{ env.PYTHON_BINARY }} || true
65+
66+
- name: Configure Homebrew for ec2-user
67+
run: |
68+
echo "Creating .brewrc file for ec2-user..."
69+
cat > /Users/ec2-user/.brewrc << 'EOF'
70+
# Homebrew environment setup
71+
export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH"
72+
export HOMEBREW_PREFIX="/opt/homebrew"
73+
export HOMEBREW_CELLAR="/opt/homebrew/Cellar"
74+
export HOMEBREW_REPOSITORY="/opt/homebrew"
75+
export HOMEBREW_NO_AUTO_UPDATE=1
76+
EOF
77+
chown ec2-user:staff /Users/ec2-user/.brewrc
78+
79+
# Fix Homebrew permissions
80+
echo "Setting permissions for Homebrew directories..."
81+
mkdir -p /opt/homebrew/Cellar
82+
chown -R ec2-user:staff /opt/homebrew
83+
shell: bash
84+
85+
- name: Install dependencies
86+
run: |
87+
echo "Installing dependencies as ec2-user..."
88+
su ec2-user -c 'source /Users/ec2-user/.brewrc && brew install lz4 automake autoconf libtool yq'
89+
shell: bash
90+
91+
- name: Checkout finch-daemon repo
92+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
93+
with:
94+
fetch-depth: 0
95+
persist-credentials: false
96+
submodules: recursive
97+
98+
- name: Configure workspace for ec2-user
99+
run: |
100+
chown -R ec2-user:staff ${{ github.workspace }}
101+
102+
- name: Install Finch
103+
run: |
104+
echo "Installing Finch as ec2-user..."
105+
su ec2-user -c 'source /Users/ec2-user/.brewrc && brew install finch --cask'
106+
su ec2-user -c 'source /Users/ec2-user/.brewrc && brew list | grep finch || echo "finch not installed"'
107+
mkdir -p /private/var/run/finch-lima
108+
cat /etc/passwd
109+
chown ec2-user:daemon /private/var/run/finch-lima
110+
shell: bash
111+
112+
- name: Build binaries
113+
run: |
114+
echo "Building cross architecture binaries..."
115+
su ec2-user -c 'cd ${{ github.workspace }} && STATIC=1 GOPROXY=direct GOOS=linux GOARCH=arm64 make'
116+
su ec2-user -c 'finch vm remove -f' || true
117+
cp -f ${{ github.workspace }}/bin/finch-daemon /Applications/Finch/finch-daemon/finch-daemon
118+
# Restart finch-daemon with new binary
119+
su ec2-user -c 'finch vm stop' || true
120+
su ec2-user -c 'finch vm start' || true
121+
shell: bash
122+
123+
- name: Check Finch version
124+
run: |
125+
echo "Initializing VM and checking version..."
126+
# Clean up any leftover network state
127+
sudo pkill -f socket_vmnet || true
128+
sudo rm -f /private/var/run/finch-lima/*.sock || true
129+
su ec2-user -c 'finch vm init'
130+
sleep 5 # Wait for services to be ready
131+
echo "Checking Finch version..."
132+
su ec2-user -c 'LIMA_HOME=/Applications/Finch/lima/data /Applications/Finch/lima/bin/limactl shell finch curl --unix-socket /var/run/finch.sock -X GET http:/v1.43/version'
133+
shell: bash
134+
135+
- name: Configure AWS credentials
136+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
137+
with:
138+
role-to-assume: ${{ secrets.SAMCLI_VM_ROLE_SYNC }}
139+
role-session-name: samcli-finch-vm-sequential-tests
140+
aws-region: ${{ secrets.REGION }}
141+
142+
- name: Install Docker CLI for SAM CLI compatibility
143+
run: |
144+
echo "Checking Docker CLI installation..."
145+
if ! su ec2-user -c 'which docker' > /dev/null 2>&1; then
146+
echo "Installing Docker CLI..."
147+
su ec2-user -c 'source /Users/ec2-user/.brewrc && brew install --formula docker'
148+
else
149+
echo "Docker CLI already installed"
150+
fi
151+
shell: bash
152+
153+
- name: Checkout SAM CLI
154+
uses: actions/checkout@v4
155+
with:
156+
repository: aws/aws-sam-cli
157+
submodules: recursive
158+
path: aws-sam-cli
159+
160+
- name: Set up SAM CLI from source
161+
run: |
162+
# Move to ec2-user home and change ownership
163+
sudo rm -rf /Users/ec2-user/aws-sam-cli || true
164+
sudo mv aws-sam-cli /Users/ec2-user/aws-sam-cli
165+
sudo chown -R ec2-user:staff /Users/ec2-user/aws-sam-cli
166+
167+
# Install and setup (use full path)
168+
su ec2-user -c 'cd /Users/ec2-user/aws-sam-cli && ${{ env.PYTHON_BINARY }} -m pip install --upgrade pip --user'
169+
su ec2-user -c 'cd /Users/ec2-user/aws-sam-cli && SAM_CLI_DEV=1 ${{ env.PYTHON_BINARY }} -m pip install -e ".[dev]" --user'
170+
su ec2-user -c 'cd /Users/ec2-user/aws-sam-cli && export PATH="/Users/ec2-user/Library/Python/${{ env.PYTHON_VERSION }}/bin:$PATH" && samdev --version'
171+
shell: bash
172+
173+
- name: Run unit tests
174+
continue-on-error: true
175+
run: ./scripts/samcli-vm/run-unit-tests.sh
176+
177+
- name: Run sync tests
178+
continue-on-error: true
179+
run: ./scripts/samcli-vm/run-sync-tests.sh
180+
181+
- name: Run package tests
182+
continue-on-error: true
183+
run: ./scripts/samcli-vm/run-package-tests.sh
184+
185+
- name: Run start-api tests
186+
continue-on-error: true
187+
run: ./scripts/samcli-vm/run-start-api-tests.sh
188+
189+
- name: Run start-lambda tests
190+
continue-on-error: true
191+
run: ./scripts/samcli-vm/run-start-lambda-tests.sh
192+
193+
- name: Patch SAM CLI for Docker image cleanup
194+
continue-on-error: true
195+
run: |
196+
# Apply git patch to handle ImageNotFound exceptions for all Docker tests
197+
su ec2-user -c 'cd /Users/ec2-user/aws-sam-cli && git apply ${{ github.workspace }}/scripts/samcli-vm/invoke-teardown.patch'
198+
shell: bash
199+
200+
- name: Run invoke tests
201+
continue-on-error: true
202+
run: ./scripts/samcli-vm/run-invoke-tests.sh
203+
204+
# ensuring resources are clean post-test
205+
cleanup:
206+
runs-on: ubuntu-latest
207+
needs: samcli-vm-test
208+
if: always()
209+
steps:
210+
- name: Checkout repository
211+
uses: actions/checkout@v4
212+
213+
- name: Configure AWS credentials
214+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df
215+
with:
216+
role-to-assume: ${{ secrets.SAMCLI_VM_ROLE_SYNC }}
217+
role-session-name: cleanup
218+
aws-region: ${{ secrets.REGION }}
219+
220+
- name: Comprehensive AWS resource cleanup
221+
timeout-minutes: 10
222+
run: ./scripts/cleanup-aws-resources.sh

‎scripts/samcli-vm/invoke-teardown.patch‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
--- a/tests/integration/local/invoke/test_integrations_cli.py
2+
+++ b/tests/integration/local/invoke/test_integrations_cli.py
3+
@@ -754,6 +754,11 @@
4+
docker_client = docker.from_env()
5+
samcli_images = docker_client.images.list(name="samcli/lambda")
6+
for image in samcli_images:
7+
- docker_client.images.remove(image.id)
8+
+ try:
9+
+ docker_client.images.remove(image.id, force=True)
10+
+ except docker.errors.ImageNotFound:
11+
+ print(f"Image {image.id} was not found. It may have been removed already.")
12+
+ except Exception as e:
13+
+ print(f"An error occurred while trying to remove image {image.id}: {str(e)}")
14+
15+
shutil.rmtree(str(self.layer_cache), ignore_errors=True)
16+
+

‎scripts/samcli-vm/run-invoke-tests.sh‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/bin/bash
2+
set -e
3+
4+
echo "=== INVOKE TESTS - Started at $(date) ==="
5+
touch /tmp/invoke_output.txt
6+
chown ec2-user:staff /tmp/invoke_output.txt
7+
8+
su ec2-user -c "
9+
cd /Users/ec2-user/aws-sam-cli && \
10+
export PATH='/Users/ec2-user/Library/Python/$PYTHON_VERSION/bin:$PATH' && \
11+
export DOCKER_HOST='$DOCKER_HOST' && \
12+
AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION' \
13+
BY_CANARY='$BY_CANARY' \
14+
SAM_CLI_DEV='$SAM_CLI_DEV' \
15+
SAM_CLI_TELEMETRY='$SAM_CLI_TELEMETRY' \
16+
'$PYTHON_BINARY' -m pytest tests/integration/local/invoke -k 'not Terraform' -v --tb=short
17+
" 2>&1 | tee /tmp/invoke_output.txt || true
18+
19+
echo ""
20+
echo "=== PASSES ==="
21+
grep "PASSED" /tmp/invoke_output.txt || echo "No passes found"
22+
23+
echo ""
24+
echo "=== FAILURES ==="
25+
grep "FAILED" /tmp/invoke_output.txt || echo "No failures found"
26+
27+
# test_invoke_with_error_during_image_build: Build error message differs from expected.
28+
# test_invoke_with_timeout_set_X_TimeoutFunction: Returns timeout message instead of empty string,
29+
# but matches actual Lambda service behavior.
30+
# test_building_new_rapid_image_removes_old_rapid_images: Cannot remove images with same digest,
31+
# Docker creates different IDs for each.
32+
# test_caching_two_layers and test_caching_two_layers_with_layer_cache_env_set: error due to sequential
33+
# test runs within invoke. Work when run in isolation and locally.
34+
# test_successful_invoke: Related to symlink mount errors due to permissions. Works locally.
35+
cat > expected_invoke_failures.txt << 'EOF'
36+
test_invoke_with_error_during_image_build
37+
test_invoke_with_timeout_set_0_TimeoutFunction
38+
test_invoke_with_timeout_set_1_TimeoutFunctionWithParameter
39+
test_invoke_with_timeout_set_2_TimeoutFunctionWithStringParameter
40+
test_building_new_rapid_image_removes_old_rapid_images
41+
test_caching_two_layers
42+
test_caching_two_layers_with_layer_cache_env_set
43+
test_successful_invoke
44+
EOF
45+
46+
# Extract actual failures
47+
grep "FAILED" /tmp/invoke_output.txt | grep -o "test_[^[:space:]]*" > actual_invoke_failures.txt || true
48+
49+
# Find unexpected failures
50+
UNEXPECTED=$(grep -v -f expected_invoke_failures.txt actual_invoke_failures.txt 2>/dev/null || true)
51+
52+
if [ -n "$UNEXPECTED" ]; then
53+
echo "❌ Unexpected failures found:"
54+
echo "$UNEXPECTED"
55+
echo ""
56+
echo "=== FULL OUTPUT FOR DEBUGGING ==="
57+
cat /tmp/invoke_output.txt
58+
exit 1
59+
else
60+
echo "✅ All failures were expected"
61+
fi
62+
63+
echo ""
64+
echo "=== PYTEST SUMMARY ==="
65+
grep -E "=+ .*(failed|passed|skipped|deselected).* =+$" /tmp/invoke_output.txt | tail -1 || echo "No pytest summary found"

‎scripts/samcli-vm/run-package-tests.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
set -e
3+
4+
echo "=== PACKAGE TESTS - Started at $(date) ==="
5+
touch /tmp/package_output.txt
6+
chown ec2-user:staff /tmp/package_output.txt
7+
su ec2-user -c "
8+
cd /Users/ec2-user/aws-sam-cli && \
9+
export PATH='/Users/ec2-user/Library/Python/$PYTHON_VERSION/bin:$PATH' && \
10+
export DOCKER_HOST='$DOCKER_HOST' && \
11+
AWS_DEFAULT_REGION='$AWS_DEFAULT_REGION' \
12+
BY_CANARY='$BY_CANARY' \
13+
SAM_CLI_DEV='$SAM_CLI_DEV' \
14+
SAM_CLI_TELEMETRY='$SAM_CLI_TELEMETRY' \
15+
'$PYTHON_BINARY' -m pytest tests/integration/package/test_package_command_image.py -v --tb=short
16+
" > /tmp/package_output.txt 2>&1 || true
17+
18+
echo ""
19+
echo "=== PASSES ==="
20+
grep "PASSED" /tmp/package_output.txt || echo "No passes found"
21+
22+
echo ""
23+
echo "=== FAILURES ==="
24+
grep "FAILED" /tmp/package_output.txt || echo "No failures found"
25+
26+
# test_package_with_deep_nested_template_image: Expects Docker-specific push stream pattern.
27+
# test_package_template_with_image_repositories_nested_stack_x: Push API stream differs from Docker.
28+
# test_package_with_loadable_image_archive_0_template_image_load_yaml: Docker imports by digest,
29+
# Finch imports as "overlayfs:" tag causing image info lookup to fail.
30+
cat > expected_package_failures.txt << 'EOF'
31+
test_package_with_deep_nested_template_image
32+
test_package_template_with_image_repositories_nested_stack
33+
test_package_with_loadable_image_archive_0_template_image_load_yaml
34+
EOF
35+
36+
# Extract actual failures
37+
grep "FAILED" /tmp/package_output.txt | grep -o "test_[^[:space:]]*" > actual_package_failures.txt || true
38+
39+
# Also check for nested stack failures (pattern match)
40+
grep "FAILED.*test_package_template_with_image_repositories_nested_stack" /tmp/package_output.txt >> actual_package_failures.txt || true
41+
42+
# Find unexpected failures (exclude nested stack pattern)
43+
UNEXPECTED=$(grep -v -f expected_package_failures.txt actual_package_failures.txt | grep -v "test_package_template_with_image_repositories_nested_stack" || true)
44+
45+
if [ -n "$UNEXPECTED" ]; then
46+
echo "❌ Unexpected failures found:"
47+
echo "$UNEXPECTED"
48+
echo ""
49+
echo "=== FULL OUTPUT FOR DEBUGGING ==="
50+
cat /tmp/package_output.txt
51+
exit 1
52+
else
53+
echo "✅ All failures were expected"
54+
fi
55+
56+
echo ""
57+
echo "=== PYTEST SUMMARY ==="
58+
grep -E "=+ .*(failed|passed|skipped|deselected).* =+$" /tmp/package_output.txt | tail -1 || echo "No pytest summary found"

0 commit comments

Comments
 (0)