Skip to content

Commit b91e941

Browse files
ayush-pantaayush-panta
authored
ci: add samcli tests directly against finch-daemon (runfinch#329)
This is an implementation of automated tests for SAMcli directly against finch-daemon in an Ubuntu environment. It runs on every PR and takes about 20 minutes to complete. Thorough testing has been done to ensure the consistency of these tests. --------- Signed-off-by: ayush-panta <ayushkp@amazon.com>
1 parent 7148cd8 commit b91e941

File tree

8 files changed

+424
-63
lines changed

8 files changed

+424
-63
lines changed

‎.github/workflows/samcli-direct.yaml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
name: samcli-direct
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
workflow_dispatch:
8+
9+
env:
10+
GO_VERSION: '1.24.x'
11+
CONTAINERD_VERSION: '2.0.x'
12+
13+
permissions:
14+
id-token: write
15+
contents: read
16+
17+
jobs:
18+
samcli-direct-test:
19+
runs-on: ubuntu-latest
20+
timeout-minutes: 30 # start-api is the longest at ~ 20 minutes
21+
strategy:
22+
fail-fast: false
23+
matrix:
24+
test_step:
25+
- name: unit
26+
- name: package
27+
- name: start-lambda
28+
- name: invoke
29+
- name: start-api
30+
env:
31+
AWS_DEFAULT_REGION: "${{ secrets.REGION }}"
32+
DOCKER_HOST: unix:///run/finch.sock
33+
DOCKER_CONFIG: $HOME/.finch
34+
BY_CANARY: true # allows full testing
35+
SAM_CLI_DEV: 1
36+
SAM_CLI_TELEMETRY: 0
37+
steps:
38+
39+
- name: Configure AWS credentials
40+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
41+
with:
42+
role-to-assume: ${{secrets.SAMCLI_DIRECT_ROLE_BASE}}
43+
role-session-name: samcli-${{ matrix.test_step.name }}-tests
44+
aws-region: ${{ secrets.REGION }}
45+
role-duration-seconds: 2000
46+
47+
- name: Set up Go
48+
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
49+
with:
50+
go-version: ${{ env.GO_VERSION }}
51+
52+
# from aws/aws-sam-cli/setup.py: python_requires=">=3.9, <=4.0, !=4.0
53+
- name: Set up Python
54+
uses: actions/setup-python@v4
55+
with:
56+
python-version: '3.11'
57+
58+
- name: Checkout finch-daemon repo
59+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
60+
61+
- name: Stop pre-existing services
62+
run: |
63+
sudo systemctl stop docker
64+
sudo systemctl stop containerd
65+
66+
- name: Remove default podman network config
67+
run: |
68+
sudo rm -f /etc/cni/net.d/87-podman-bridge.conflist
69+
70+
- name: Clean up Daemon socket
71+
run: |
72+
sudo rm -f /run/finch.sock
73+
sudo rm -f /run/finch.pid
74+
sudo rm -f /run/finch-credential.sock
75+
76+
- name: Install finch-daemon dependencies
77+
run: |
78+
./setup-test-env.sh
79+
sleep 10
80+
81+
- name: Build and start finch-daemon
82+
run: |
83+
make build
84+
sudo bin/finch-daemon --debug --socket-owner $UID 2>&1 | tee finch-daemon.log &
85+
sleep 10
86+
87+
- name: Checkout SAM CLI
88+
uses: actions/checkout@v4
89+
with:
90+
repository: aws/aws-sam-cli
91+
submodules: recursive
92+
path: aws-sam-cli
93+
94+
- name: Set up SAM CLI from source
95+
working-directory: aws-sam-cli
96+
run: |
97+
python -m pip install --upgrade pip
98+
make init
99+
samdev --version
100+
101+
- name: Run unit tests
102+
if: matrix.test_step.name == 'unit'
103+
run: ./scripts/samcli-direct/run-unit-tests.sh
104+
105+
- name: Run package tests
106+
if: matrix.test_step.name == 'package'
107+
run: ./scripts/samcli-direct/run-package-tests.sh
108+
109+
- name: Run invoke tests
110+
if: matrix.test_step.name == 'invoke'
111+
run: ./scripts/samcli-direct/run-invoke-tests.sh
112+
113+
- name: Run start-lambda tests
114+
if: matrix.test_step.name == 'start-lambda'
115+
run: ./scripts/samcli-direct/run-start-lambda-tests.sh
116+
117+
- name: Run start-api tests
118+
if: matrix.test_step.name == 'start-api'
119+
run: ./scripts/samcli-direct/run-start-api-tests.sh
120+
121+
- name: Show finch-daemon logs
122+
if: always()
123+
run: |
124+
echo "=== FINCH-DAEMON OUTPUT ==="
125+
cat finch-daemon.log
126+
127+
# ensuring resources are clean post-test
128+
cleanup:
129+
runs-on: ubuntu-latest
130+
needs: samcli-direct-test
131+
if: always()
132+
steps:
133+
- name: Checkout repository
134+
uses: actions/checkout@v4
135+
136+
- name: Configure AWS credentials
137+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df
138+
with:
139+
role-to-assume: ${{ secrets.SAMCLI_DIRECT_ROLE_BASE }}
140+
role-session-name: cleanup-samcli-direct
141+
aws-region: ${{ secrets.REGION }}
142+
143+
- name: Comprehensive AWS resource cleanup
144+
timeout-minutes: 10
145+
run: ./scripts/cleanup-aws-resources.sh

‎scripts/cleanup-aws-resources.sh‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
set +e # Continue on failures
3+
4+
echo "=== AWS Resource Cleanup ==="
5+
6+
# Function to safely run AWS commands with retries
7+
safe_aws_command() {
8+
local max_attempts=3
9+
local attempt=1
10+
local command="$@"
11+
while [ $attempt -le $max_attempts ]; do
12+
if eval "$command"; then
13+
return 0
14+
fi
15+
echo "Retry $attempt/$max_attempts failed: $command"
16+
sleep 5
17+
attempt=$((attempt + 1))
18+
done
19+
echo "Command failed after $max_attempts attempts: $command"
20+
return 1
21+
}
22+
23+
# Clean up S3 buckets from SAM CLI test stacks
24+
echo "=== Cleaning S3 buckets ==="
25+
TEST_PATTERNS=("sam-app" "test-" "integration-test" "samcli" "aws-sam-cli-managed")
26+
27+
for pattern in "${TEST_PATTERNS[@]}"; do
28+
STACKS=$(aws cloudformation list-stacks --region $AWS_DEFAULT_REGION --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE ROLLBACK_COMPLETE UPDATE_ROLLBACK_COMPLETE --query "StackSummaries[?contains(StackName, '$pattern')].[StackName]" --output text 2>/dev/null || true)
29+
30+
for stack in $STACKS; do
31+
echo "Processing stack: $stack"
32+
33+
# Get S3 buckets from stack
34+
BUCKET_NAMES=$(aws cloudformation describe-stacks --stack-name "$stack" --region $AWS_DEFAULT_REGION --query 'Stacks[0].Outputs[?contains(OutputKey, `Bucket`) || contains(OutputKey, `bucket`)].OutputValue' --output text 2>/dev/null || true)
35+
RESOURCE_BUCKETS=$(aws cloudformation describe-stack-resources --stack-name "$stack" --region $AWS_DEFAULT_REGION --query 'StackResources[?ResourceType==`AWS::S3::Bucket`].PhysicalResourceId' --output text 2>/dev/null || true)
36+
37+
# Empty buckets (don't delete them)
38+
for bucket in $BUCKET_NAMES $RESOURCE_BUCKETS; do
39+
if [ -n "$bucket" ] && [ "$bucket" != "None" ]; then
40+
echo "Emptying S3 bucket: $bucket"
41+
if aws s3api head-bucket --bucket "$bucket" 2>/dev/null; then
42+
safe_aws_command "aws s3 rm s3://$bucket --recursive --quiet" || true
43+
echo "✅ Emptied bucket: $bucket"
44+
fi
45+
fi
46+
done
47+
done
48+
done
49+
50+
# Clean up ECR repositories
51+
echo "=== Cleaning ECR repositories ==="
52+
ECR_PATTERNS=("sam-app" "test-" "integration-test")
53+
for pattern in "${ECR_PATTERNS[@]}"; do
54+
REPOS=$(aws ecr describe-repositories --region $AWS_DEFAULT_REGION --query "repositories[?contains(repositoryName, '$pattern')].repositoryName" --output text 2>/dev/null || true)
55+
for repo in $REPOS; do
56+
echo "Deleting ECR repository: $repo"
57+
safe_aws_command "aws ecr delete-repository --repository-name '$repo' --force --region $AWS_DEFAULT_REGION" || true
58+
done
59+
done
60+
61+
echo "✅ Cleanup completed"

‎scripts/samcli-direct/run-invoke-tests.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
set -e
3+
4+
cd aws-sam-cli
5+
6+
python -m pytest tests/integration/local/invoke -k 'not Terraform' -v --tb=short > invoke_output.txt 2>&1 || true
7+
8+
echo ""
9+
echo "=== PASSES ==="
10+
grep "PASSED" invoke_output.txt || echo "No passes found"
11+
12+
echo ""
13+
echo "=== FAILURES ==="
14+
grep "FAILED" invoke_output.txt || echo "No failures found"
15+
16+
# test_invoke_with_error_during_image_build: Build error message differs from expected.
17+
# test_invoke_with_timeout_set_X_TimeoutFunction: Returns timeout message instead of empty string,
18+
# but matches actual Lambda service behavior.
19+
# test_building_new_rapid_image_removes_old_rapid_images: Cannot remove images with same digest,
20+
# Docker creates different IDs for each.
21+
cat > expected_invoke_failures.txt << 'EOF'
22+
test_invoke_with_error_during_image_build
23+
test_invoke_with_timeout_set_0_TimeoutFunction
24+
test_invoke_with_timeout_set_1_TimeoutFunctionWithParameter
25+
test_invoke_with_timeout_set_2_TimeoutFunctionWithStringParameter
26+
test_building_new_rapid_image_removes_old_rapid_images
27+
EOF
28+
29+
# Extract actual failures
30+
grep "FAILED" invoke_output.txt | grep -o "test_[^[:space:]]*" > actual_invoke_failures.txt || true
31+
32+
# Find unexpected failures
33+
UNEXPECTED=$(grep -v -f expected_invoke_failures.txt actual_invoke_failures.txt 2>/dev/null || true)
34+
35+
if [ -n "$UNEXPECTED" ]; then
36+
echo "❌ Unexpected failures found:"
37+
echo "$UNEXPECTED"
38+
echo ""
39+
echo "=== FULL OUTPUT FOR DEBUGGING ==="
40+
cat invoke_output.txt || echo "No output file found"
41+
exit 1
42+
else
43+
echo "✅ All failures were expected."
44+
fi
45+
46+
echo ""
47+
echo "=== PYTEST SUMMARY ==="
48+
grep -E "=+ .*(failed|passed|skipped|deselected).* =+$" invoke_output.txt | tail -1 || echo "No pytest summary found"

‎scripts/samcli-direct/run-package-tests.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
set -e
3+
4+
cd aws-sam-cli
5+
6+
python -m pytest tests/integration/package/test_package_command_image.py -v --tb=short > package_output.txt 2>&1 || true
7+
8+
echo ""
9+
echo "=== PASSES ==="
10+
grep "PASSED" package_output.txt || echo "No passes found"
11+
12+
echo ""
13+
echo "=== FAILURES ==="
14+
grep "FAILED" package_output.txt || echo "No failures found"
15+
16+
# test_package_with_deep_nested_template_image: Expects Docker-specific push stream pattern.
17+
# test_package_template_with_image_repositories_nested_stack_x: Push API stream differs from Docker.
18+
# test_package_with_loadable_image_archive_0_template_image_load_yaml: Docker imports by digest,
19+
# Finch imports as "overlayfs:" tag causing image info lookup to fail.
20+
cat > expected_package_failures.txt << 'EOF'
21+
test_package_with_deep_nested_template_image
22+
test_package_template_with_image_repositories_nested_stack
23+
test_package_with_loadable_image_archive_0_template_image_load_yaml
24+
EOF
25+
26+
# Extract actual failures
27+
grep "FAILED" package_output.txt | grep -o "test_[^[:space:]]*" > actual_package_failures.txt || true
28+
29+
# Also check for nested stack failures (pattern match)
30+
grep "FAILED.*test_package_template_with_image_repositories_nested_stack" package_output.txt >> actual_package_failures.txt || true
31+
32+
# Find unexpected failures (exclude nested stack pattern)
33+
UNEXPECTED=$(grep -v -f expected_package_failures.txt actual_package_failures.txt | grep -v "test_package_template_with_image_repositories_nested_stack" || true)
34+
35+
if [ -n "$UNEXPECTED" ]; then
36+
echo "❌ Unexpected failures found:"
37+
echo "$UNEXPECTED"
38+
echo ""
39+
echo "=== FULL OUTPUT FOR DEBUGGING ==="
40+
cat package_output.txt || echo "No output file found"
41+
exit 1
42+
else
43+
echo "✅ All failures were expected."
44+
fi
45+
46+
echo ""
47+
echo "=== PYTEST SUMMARY ==="
48+
grep -E "=+ .*(failed|passed|skipped|deselected).* =+$" package_output.txt | tail -1 || echo "No pytest summary found"

‎scripts/samcli-direct/run-start-api-tests.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
set -e
3+
4+
cd aws-sam-cli
5+
6+
ulimit -n 65536
7+
python -m pytest tests/integration/local/start_api -k 'not Terraform' -v --tb=short > start_api_output.txt 2>&1 || true
8+
9+
echo ""
10+
echo "=== PASSES ==="
11+
grep "PASSED" start_api_output.txt || echo "No passes found"
12+
13+
echo ""
14+
echo "=== FAILURES ==="
15+
grep "FAILED" start_api_output.txt || echo "No failures found"
16+
17+
# test_can_invoke_lambda_layer_successfully: Uses random port, fails occasionally.
18+
# Only 1 test of 386 total, acceptable failure rate.
19+
cat > expected_start_api_failures.txt << 'EOF'
20+
test_can_invoke_lambda_layer_successfully
21+
EOF
22+
23+
# Extract actual failures - find test names in FAILED lines
24+
grep "FAILED" start_api_output.txt | grep -o "test_[^[:space:]]*" > actual_start_api_failures.txt || true
25+
26+
# Find unexpected failures
27+
UNEXPECTED=$(grep -v -f expected_start_api_failures.txt actual_start_api_failures.txt 2>/dev/null || true)
28+
29+
if [ -n "$UNEXPECTED" ]; then
30+
echo "❌ Unexpected start-api failures found:"
31+
echo "$UNEXPECTED"
32+
echo ""
33+
echo "=== FULL OUTPUT FOR DEBUGGING ==="
34+
cat start_api_output.txt || echo "No output file found"
35+
exit 1
36+
else
37+
echo "✅ All start-api failures (if any) were expected."
38+
fi
39+
40+
echo ""
41+
echo "=== PYTEST SUMMARY ==="
42+
grep -E "=+ .*(failed|passed|skipped|deselected).* =+$" start_api_output.txt | tail -1 || echo "No pytest summary found"

‎scripts/samcli-direct/run-start-lambda-tests.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
set -e
3+
4+
cd aws-sam-cli
5+
6+
python -m pytest tests/integration/local/start_lambda -k 'not Terraform' -v --tb=short > start_lambda_output.txt 2>&1 || true
7+
8+
echo ""
9+
echo "=== PASSES ==="
10+
grep "PASSED" start_lambda_output.txt || echo "No passes found"
11+
12+
echo ""
13+
echo "=== FAILURES ==="
14+
grep "FAILED" start_lambda_output.txt || echo "No failures found"
15+
16+
# Should pass completely per test guide
17+
if grep -q "FAILED" start_lambda_output.txt; then
18+
echo "❌ Start-lambda tests failed (should pass completely)"
19+
grep "FAILED" start_lambda_output.txt
20+
echo ""
21+
echo "=== FULL OUTPUT FOR DEBUGGING ==="
22+
cat start_lambda_output.txt || echo "No output file found"
23+
exit 1
24+
else
25+
echo "✅ All start-lambda tests passed as expected"
26+
fi
27+
28+
echo ""
29+
echo "=== PYTEST SUMMARY ==="
30+
grep -E "=+ .*(failed|passed|skipped|deselected).* =+$" start_lambda_output.txt | tail -1 || echo "No pytest summary found"

0 commit comments

Comments
 (0)