Skip to content

Commit 1b9ce8e

Browse files
nhinschnhinsch
authored
Run Integration Tests in GitHub (#121)
1 parent 7ddaad2 commit 1b9ce8e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+299
-893
lines changed

‎.github/workflows/build.yml‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ jobs:
3232
source venv/bin/activate
3333
flake8 datadog_lambda/
3434
35-
test:
35+
unit-test:
3636
runs-on: ubuntu-latest
3737
strategy:
3838
max-parallel: 4
@@ -59,3 +59,38 @@ jobs:
5959
run: |
6060
source venv/bin/activate
6161
nose2 -v
62+
63+
integration-test:
64+
runs-on: ubuntu-latest
65+
66+
steps:
67+
- name: Checkout
68+
uses: actions/checkout@v2
69+
70+
- name: Set up Node 14
71+
uses: actions/setup-node@v1
72+
with:
73+
node-version: 14
74+
75+
- name: Cache Node modules
76+
id: cache-node-modules
77+
uses: actions/cache@v2
78+
with:
79+
path: "**/node_modules"
80+
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
81+
82+
- name: Install Serverless Framework
83+
run: sudo yarn global add serverless --prefix /usr/local
84+
85+
- name: Install dependencies
86+
if: steps.cache-node-modules.outputs.cache-hit != 'true'
87+
working-directory: tests/integration
88+
run: yarn install
89+
90+
- name: Run tests
91+
env:
92+
BUILD_LAYERS: true
93+
DD_API_KEY: ${{ secrets.DD_API_KEY }}
94+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
95+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
96+
run: ./scripts/run_integration_tests.sh

‎datadog_lambda/patch.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,8 @@ def _print_request_string(request):
145145

146146
# Sort the datapoints POSTed by their name so that snapshots always align
147147
data = request.body or "{}"
148+
# If payload is compressed, decompress it so we can parse it
148149
if request.headers.get("Content-Encoding") == "deflate":
149-
# See metric.py: lambda_stats = ThreadStats(compress_payload=True)
150150
data = zlib.decompress(data)
151151
data_dict = json.loads(data)
152152
data_dict.get("series", []).sort(key=lambda series: series.get("metric"))

‎scripts/run_integration_tests.sh‎

Lines changed: 121 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,15 @@ set -e
1212
# defined for every handler_runtime combination
1313
LAMBDA_HANDLERS=("async-metrics" "sync-metrics" "http-requests" "http-error")
1414
RUNTIMES=("python27" "python36" "python37" "python38")
15-
CONFIGS=("with-plugin" "without-plugin")
1615

1716
LOGS_WAIT_SECONDS=20
1817

19-
# Force cold start to avoid flaky tests
20-
export COLD_START_ENFORCER=$((1 + $RANDOM % 100000))
21-
2218
script_path=${BASH_SOURCE[0]}
2319
scripts_dir=$(dirname $script_path)
2420
repo_dir=$(dirname $scripts_dir)
2521
integration_tests_dir="$repo_dir/tests/integration"
2622

27-
script_start_time=$(date --iso-8601=seconds)
23+
script_utc_start_time=$(date -u +"%Y%m%dT%H%M%S")
2824

2925
mismatch_found=false
3026

@@ -46,62 +42,49 @@ fi
4642

4743
cd $integration_tests_dir
4844

49-
# Install the specified plugin version
50-
yarn install
51-
5245
input_event_files=$(ls ./input_events)
5346
# Sort event files by name so that snapshots stay consistent
5447
input_event_files=($(for file_name in ${input_event_files[@]}; do echo $file_name; done | sort))
5548

56-
echo "Deploying functions with plugin"
57-
serverless deploy -c "./serverless-plugin.yml"
58-
echo "Deploying functions without plugin"
59-
serverless deploy
49+
# Generate a random 8-character ID to avoid collisions with other runs
50+
run_id=$(xxd -l 4 -c 4 -p < /dev/random)
51+
52+
echo "Deploying functions"
53+
serverless deploy --stage $run_id
6054

6155
echo "Invoking functions"
6256
set +e # Don't exit this script if an invocation fails or there's a diff
63-
for _sls_type in "${CONFIGS[@]}"; do
64-
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
65-
for runtime in "${RUNTIMES[@]}"; do
66-
if [ "$_sls_type" = "with-plugin" ]; then
67-
function_name="${handler_name}_${runtime}_with_plugin"
57+
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
58+
for runtime in "${RUNTIMES[@]}"; do
59+
function_name="${handler_name}_${runtime}"
60+
61+
# Invoke function once for each input event
62+
for input_event_file in "${input_event_files[@]}"; do
63+
# Get event name without trailing ".json" so we can build the snapshot file name
64+
input_event_name=$(echo "$input_event_file" | sed "s/.json//")
65+
snapshot_path="./snapshots/return_values/${function_name}_${input_event_name}.json"
66+
67+
return_value=$(serverless invoke -f $function_name --stage $run_id --path "./input_events/$input_event_file")
68+
69+
if [ ! -f $snapshot_path ]; then
70+
# If the snapshot file doesn't exist yet, we create it
71+
echo "Writing return value to $snapshot_path because no snapshot exists yet"
72+
echo "$return_value" >$snapshot_path
73+
elif [ -n "$UPDATE_SNAPSHOTS" ]; then
74+
# If $UPDATE_SNAPSHOTS is set to true, write the new logs over the current snapshot
75+
echo "Overwriting return value snapshot for $snapshot_path"
76+
echo "$return_value" >$snapshot_path
6877
else
69-
function_name="${handler_name}_${runtime}"
70-
fi
71-
72-
# Invoke function once for each input event
73-
for input_event_file in "${input_event_files[@]}"; do
74-
# Get event name without trailing ".json" so we can build the snapshot file name
75-
input_event_name=$(echo "$input_event_file" | sed "s/.json//")
76-
# Return value snapshot file format is snapshots/return_values/{handler}_{runtime}_{input-event}
77-
snapshot_path="./snapshots/return_values/${handler_name}_${runtime}_${input_event_name}.json"
78-
79-
if [ "$_sls_type" = "with-plugin" ]; then
80-
return_value=$(serverless invoke -f $function_name --path "./input_events/$input_event_file" -c "serverless-plugin.yml")
81-
else
82-
return_value=$(serverless invoke -f $function_name --path "./input_events/$input_event_file")
83-
fi
84-
85-
if [ ! -f $snapshot_path ]; then
86-
# If the snapshot file doesn't exist yet, we create it
87-
echo "Writing return value to $snapshot_path because no snapshot exists yet"
88-
echo "$return_value" >$snapshot_path
89-
elif [ -n "$UPDATE_SNAPSHOTS" ]; then
90-
# If $UPDATE_SNAPSHOTS is set to true, write the new logs over the current snapshot
91-
echo "Overwriting return value snapshot for $snapshot_path"
92-
echo "$return_value" >$snapshot_path
78+
# Compare new return value to snapshot
79+
diff_output=$(echo "$return_value" | diff - $snapshot_path)
80+
if [ $? -eq 1 ]; then
81+
echo "Failed: Return value for $function_name does not match snapshot:"
82+
echo "$diff_output"
83+
mismatch_found=true
9384
else
94-
# Compare new return value to snapshot
95-
diff_output=$(echo "$return_value" | diff - $snapshot_path)
96-
if [ $? -eq 1 ]; then
97-
echo "Failed: Return value for $function_name does not match snapshot:"
98-
echo "$diff_output"
99-
mismatch_found=true
100-
else
101-
echo "Ok: Return value for $function_name with $input_event_name event matches snapshot"
102-
fi
85+
echo "Ok: Return value for $function_name with $input_event_name event matches snapshot"
10386
fi
104-
done
87+
fi
10588
done
10689
done
10790
done
@@ -110,82 +93,100 @@ set -e
11093
echo "Sleeping $LOGS_WAIT_SECONDS seconds to wait for logs to appear in CloudWatch..."
11194
sleep $LOGS_WAIT_SECONDS
11295

96+
set +e # Don't exit this script if there is a diff or the logs endpoint fails
11397
echo "Fetching logs for invocations and comparing to snapshots"
114-
for _sls_type in "${CONFIGS[@]}"; do
115-
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
116-
for runtime in "${RUNTIMES[@]}"; do
117-
if [ "$_sls_type" = "with-plugin" ]; then
118-
function_name="${handler_name}_${runtime}_with_plugin"
119-
else
120-
function_name="${handler_name}_${runtime}"
121-
fi
122-
123-
function_snapshot_path="./snapshots/logs/$function_name.log"
124-
125-
# Fetch logs with serverless cli
126-
if [ "$_sls_type" = "with-plugin" ]; then
127-
raw_logs=$(serverless logs -f $function_name --startTime $script_start_time -c "serverless-plugin.yml")
128-
else
129-
raw_logs=$(serverless logs -f $function_name --startTime $script_start_time)
98+
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
99+
for runtime in "${RUNTIMES[@]}"; do
100+
function_name="${handler_name}_${runtime}"
101+
function_snapshot_path="./snapshots/logs/$function_name.log"
102+
103+
# Fetch logs with serverless cli, retrying to avoid AWS account-wide rate limit error
104+
retry_counter=0
105+
while [ $retry_counter -lt 10 ]; do
106+
raw_logs=$(serverless logs -f $function_name --stage $run_id --startTime $script_utc_start_time)
107+
fetch_logs_exit_code=$?
108+
if [ $fetch_logs_exit_code -eq 1 ]; then
109+
echo "Retrying fetch logs for $function_name..."
110+
retry_counter=$(($retry_counter + 1))
111+
sleep 10
112+
continue
130113
fi
114+
break
115+
done
131116

132-
# Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions
133-
logs=$(
134-
echo "$raw_logs" |
135-
# Filter serverless cli errors
136-
sed '/Serverless: Recoverable error occurred/d' |
137-
# Remove RequestsDependencyWarning from botocore/vendored/requests/__init__.py
138-
sed '/RequestsDependencyWarning/d' |
139-
# Remove blank lines
140-
sed '/^$/d' |
141-
# Normalize Lambda runtime report logs
142-
sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"): [a-z0-9\.\-]+/\1: XXXX/g' |
143-
# Normalize DD APM headers and AWS account ID
144-
sed -E "s/(x-datadog-parent-id:|x-datadog-trace-id:|account_id:)[0-9]+/\1XXXX/g" |
145-
# Normalize timestamps in datapoints POSTed to DD
146-
sed -E 's/"points": \[\[[0-9\.]+,/"points": \[\[XXXX,/g' |
147-
# Strip API key from logged requests
148-
sed -E "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" |
149-
# Normalize minor package version so that these snapshots aren't broken on version bumps
150-
sed -E "s/(dd_lambda_layer:datadog-python[0-9]+_2\.)[0-9]+\.0/\1XX\.0/g" |
151-
sed -E "s/(datadog_lambda:v)([0-9]+\.[0-9]+\.[0-9])/\1XX/g" |
152-
# Strip out trace/span/parent/timestamps
153-
sed -E "s/(\"trace_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
154-
sed -E "s/(\"span_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
155-
sed -E "s/(\"parent_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
156-
sed -E "s/(\"request_id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
157-
sed -E "s/(\"duration\"\: )[0-9\.\-]+/\1XXXX/g" |
158-
sed -E "s/(\"start\"\: )[0-9\.\-]+/\1XXXX/g" |
159-
sed -E "s/(\"system\.pid\"\: )[0-9\.\-]+/\1XXXX/g" |
160-
sed -E "s/(\"runtime-id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
161-
sed -E "s/(\"datadog_lambda\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g" |
162-
sed -E "s/(\"dd_trace\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g"
163-
)
164-
165-
if [ ! -f $function_snapshot_path ]; then
166-
# If no snapshot file exists yet, we create one
167-
echo "Writing logs to $function_snapshot_path because no snapshot exists yet"
168-
echo "$logs" >$function_snapshot_path
169-
elif [ -n "$UPDATE_SNAPSHOTS" ]; then
170-
# If $UPDATE_SNAPSHOTS is set to true write the new logs over the current snapshot
171-
echo "Overwriting log snapshot for $function_snapshot_path"
172-
echo "$logs" >$function_snapshot_path
117+
if [ $retry_counter -eq 9 ]; then
118+
echo "FAILURE: Could not retrieve logs for $function_name"
119+
echo "Error from final attempt to retrieve logs:"
120+
echo $raw_logs
121+
122+
echo "Removing functions"
123+
serverless remove --stage $run_id
124+
125+
exit 1
126+
fi
127+
128+
# Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions
129+
logs=$(
130+
echo "$raw_logs" |
131+
# Filter serverless cli errors
132+
sed '/Serverless: Recoverable error occurred/d' |
133+
# Remove RequestsDependencyWarning from botocore/vendored/requests/__init__.py
134+
sed '/RequestsDependencyWarning/d' |
135+
# Remove blank lines
136+
sed '/^$/d' |
137+
# Normalize Lambda runtime REPORT logs
138+
sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"): [a-z0-9\.\-]+/\1: XXXX/g' |
139+
# Normalize HTTP headers
140+
sed -E "s/(x-datadog-parent-id:|x-datadog-trace-id:|Content-Length:)[0-9]+/\1XXXX/g" |
141+
# Remove Account ID
142+
sed -E "s/(account_id:)[0-9]+/\1XXXX/g" |
143+
# Normalize timestamps in datapoints POSTed to DD
144+
sed -E 's/"points": \[\[[0-9\.]+,/"points": \[\[XXXX,/g' |
145+
# Strip API key from logged requests
146+
sed -E "s/(api_key=|'api_key': ')[a-z0-9\.\-]+/\1XXXX/g" |
147+
# Normalize minor package version so that these snapshots aren't broken on version bumps
148+
sed -E "s/(dd_lambda_layer:datadog-python[0-9]+_)[0-9]+\.[0-9]+\.[0-9]+/\1X\.X\.X/g" |
149+
sed -E "s/(datadog_lambda:v)([0-9]+\.[0-9]+\.[0-9])/\1XX/g" |
150+
# Strip out run ID (from function name, resource, etc.)
151+
sed -E "s/$run_id/XXXX/g" |
152+
# Strip out trace/span/parent/timestamps
153+
sed -E "s/(\"trace_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
154+
sed -E "s/(\"span_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
155+
sed -E "s/(\"parent_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
156+
sed -E "s/(\"request_id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
157+
sed -E "s/(\"duration\"\: )[0-9\.\-]+/\1XXXX/g" |
158+
sed -E "s/(\"start\"\: )[0-9\.\-]+/\1XXXX/g" |
159+
sed -E "s/(\"system\.pid\"\: )[0-9\.\-]+/\1XXXX/g" |
160+
sed -E "s/(\"runtime-id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
161+
sed -E "s/(\"datadog_lambda\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g" |
162+
sed -E "s/(\"dd_trace\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g"
163+
)
164+
165+
if [ ! -f $function_snapshot_path ]; then
166+
# If no snapshot file exists yet, we create one
167+
echo "Writing logs to $function_snapshot_path because no snapshot exists yet"
168+
echo "$logs" >$function_snapshot_path
169+
elif [ -n "$UPDATE_SNAPSHOTS" ]; then
170+
# If $UPDATE_SNAPSHOTS is set to true write the new logs over the current snapshot
171+
echo "Overwriting log snapshot for $function_snapshot_path"
172+
echo "$logs" >$function_snapshot_path
173+
else
174+
# Compare new logs to snapshots
175+
diff_output=$(echo "$logs" | diff - $function_snapshot_path)
176+
if [ $? -eq 1 ]; then
177+
echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):"
178+
echo "$diff_output"
179+
mismatch_found=true
173180
else
174-
# Compare new logs to snapshots
175-
set +e # Don't exit this script if there is a diff
176-
diff_output=$(echo "$logs" | diff - $function_snapshot_path)
177-
if [ $? -eq 1 ]; then
178-
echo "Failed: Mismatch found between new $function_name logs (first) and snapshot (second):"
179-
echo "$diff_output"
180-
mismatch_found=true
181-
else
182-
echo "Ok: New logs for $function_name match snapshot"
183-
fi
184-
set -e
181+
echo "Ok: New logs for $function_name match snapshot"
185182
fi
186-
done
183+
fi
187184
done
188185
done
186+
set -e
187+
188+
echo "Removing functions"
189+
serverless remove --stage $run_id
189190

190191
if [ "$mismatch_found" = true ]; then
191192
echo "FAILURE: A mismatch between new data and a snapshot was found and printed above."

‎tests/integration/decorator.py‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎tests/integration/handle.py‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,8 @@
1-
import os
2-
3-
from decorator import conditional_decorator
41
from datadog_lambda.metric import lambda_metric
52
from datadog_lambda.wrapper import datadog_lambda_wrapper
63

7-
with_plugin = os.getenv("WITH_PLUGIN", False)
8-
94

10-
@conditional_decorator(datadog_lambda_wrapper, with_plugin)
5+
@datadog_lambda_wrapper
116
def handle(event, context):
127
# Parse request ID and record ids out of the event to include in the response
138
request_id = event.get("requestContext", {}).get("requestId")

‎tests/integration/http_error.py‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,10 @@
1-
import os
21
import requests
32

4-
from decorator import conditional_decorator
53
from datadog_lambda.metric import lambda_metric
64
from datadog_lambda.wrapper import datadog_lambda_wrapper
7-
from ddtrace import tracer
8-
from ddtrace.internal.writer import LogWriter
95

10-
tracer.writer = LogWriter()
11-
with_plugin = os.getenv("WITH_PLUGIN", False)
126

13-
14-
@conditional_decorator(datadog_lambda_wrapper, with_plugin)
7+
@datadog_lambda_wrapper
158
def handle(event, context):
169
lambda_metric("hello.dog", 1, tags=["team:serverless", "role:hello"])
1710
lambda_metric(

0 commit comments

Comments
 (0)