From ab99f6de2a1b85aa744050ead363974110a081d6 Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Mon, 3 Nov 2025 15:29:18 -0500 Subject: [PATCH 1/7] Increase load benchmark repetition count to 5 --- benchmark/load/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 5f2f265b045..b026f88f552 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -43,10 +43,10 @@ for app in *; do # Using profiler variants for healthcheck as they are the slowest if [ "${app}" == "petclinic" ]; then HEALTHCHECK_URL=http://localhost:8082 - REPETITIONS_COUNT=2 + REPETITIONS_COUNT=5 elif [ "${app}" == "insecure-bank" ]; then HEALTHCHECK_URL=http://localhost:8082/login - REPETITIONS_COUNT=2 + REPETITIONS_COUNT=5 else echo "Unknown app ${app}" exit 1 From fb44c5d4bae5116e83a5c0c2760417ef37e457ba Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Mon, 3 Nov 2025 17:07:54 -0500 Subject: [PATCH 2/7] Increase timeout to 2h --- .gitlab/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index b2231e75f8d..a3a247d1959 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -1,7 +1,7 @@ .benchmarks: stage: benchmarks interruptible: true - timeout: 1h + timeout: 2h tags: ["runner:apm-k8s-tweaked-metal"] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-java-benchmarks needs: [ "build", "publish-artifacts-to-s3" ] From 59bfbdab352e07e36a176bfc1538aad2bc6cb5cc Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Thu, 6 Nov 2025 11:04:56 -0500 Subject: [PATCH 3/7] Attempt to parallelize load benchmarks --- benchmark/load/insecure-bank/k6.js | 12 +++--- benchmark/load/insecure-bank/start-servers.sh | 12 +++--- benchmark/load/run.sh | 39 ++++++++++++------- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 2dd800fa7e5..77ae5be643d 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -3,22 +3,22 @@ import {checkResponse, isOk, isRedirect} from "../../utils/k6.js"; const variants = { "no_agent": { - "APP_URL": 'http://localhost:8080', + "APP_URL": 'http://localhost:8086', }, "tracing": { - "APP_URL": 'http://localhost:8081', + "APP_URL": 'http://localhost:8087', }, "profiling": { - "APP_URL": 'http://localhost:8082', + "APP_URL": 'http://localhost:8088', }, "iast": { - "APP_URL": 'http://localhost:8083', + "APP_URL": 'http://localhost:8089', }, "iast_GLOBAL": { - "APP_URL": 'http://localhost:8084', + "APP_URL": 'http://localhost:8090', }, "iast_FULL": { - "APP_URL": 'http://localhost:8085', + "APP_URL": 'http://localhost:8091', }, } diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index 4cae95567f2..92c423cdc4d 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -18,11 +18,11 @@ start_server() { echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log [PID=$PID]" } -start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " & -start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " & -start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " & -start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " & -start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " & -start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " & +start_server "no_agent" "-Dserver.port=8086" "taskset -c 48 " & +start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8087" "taskset -c 49 " & +start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8088" "taskset -c 50 " & +start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8089" "taskset -c 51 " & +start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8090" "taskset -c 52 " & +start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8091" "taskset -c 53 " & wait diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index b026f88f552..b00e919473a 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -27,11 +27,9 @@ fi source "${UTILS_DIR}/update-java-version.sh" 17 -for app in *; do - if [[ ! -d "${app}" ]]; then - continue - fi - +run_app_benchmark() { + local app=$1 + message "${type} benchmark: ${app} started" export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" @@ -44,9 +42,15 @@ for app in *; do if [ "${app}" == "petclinic" ]; then HEALTHCHECK_URL=http://localhost:8082 REPETITIONS_COUNT=5 + PORT_START=8080 + PORT_END=8085 + HEALTHCHECK_PATH="" elif [ "${app}" == "insecure-bank" ]; then - HEALTHCHECK_URL=http://localhost:8082/login + HEALTHCHECK_URL=http://localhost:8088/login REPETITIONS_COUNT=5 + PORT_START=8086 + PORT_END=8091 + HEALTHCHECK_PATH="/login" else echo "Unknown app ${app}" exit 1 @@ -56,15 +60,9 @@ for app in *; do bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & echo "Waiting for serves to start..." - if [ "${app}" == "petclinic" ]; then - for port in $(seq 8080 8085); do - healthcheck http://localhost:$port - done - elif [ "${app}" == "insecure-bank" ]; then - for port in $(seq 8080 8085); do - healthcheck http://localhost:$port/login - done - fi + for port in $(seq $PORT_START $PORT_END); do + healthcheck http://localhost:$port$HEALTHCHECK_PATH + done echo "Servers are up!" ( @@ -74,4 +72,15 @@ for app in *; do done message "${type} benchmark: ${app} finished" +} + +# Run petclinic and insecure-bank benchmarks in parallel to reduce total runtime +for app in *; do + if [[ ! -d "${app}" ]]; then + continue + fi + run_app_benchmark "${app}" & done + +# Wait for all background jobs to complete +wait From a9a53c669bcb6d6f0c5434b0667c3486f4d10cfc Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Fri, 7 Nov 2025 10:08:06 -0500 Subject: [PATCH 4/7] Parallelize into separate jobs --- .gitlab/benchmarks.yml | 16 +++++-- benchmark/load/insecure-bank/k6.js | 12 ++--- benchmark/load/insecure-bank/start-servers.sh | 12 ++--- benchmark/load/run.sh | 47 +++++++++---------- 4 files changed, 48 insertions(+), 39 deletions(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index a3a247d1959..11ec5c6b6ac 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -35,12 +35,20 @@ benchmarks-startup: - ./steps/run-benchmarks.sh startup - ./steps/analyze-results.sh startup -benchmarks-load: +benchmarks-load-petclinic: extends: .benchmarks script: - !reference [ .benchmarks, script ] - ./steps/capture-hardware-software-info.sh - - ./steps/run-benchmarks.sh load + - ./steps/run-benchmarks.sh load petclinic + - ./steps/analyze-results.sh load + +benchmarks-load-insecure-bank: + extends: .benchmarks + script: + - !reference [ .benchmarks, script ] + - ./steps/capture-hardware-software-info.sh + - ./steps/run-benchmarks.sh load insecure-bank - ./steps/analyze-results.sh load benchmarks-dacapo: @@ -61,7 +69,9 @@ benchmarks-post-results: needs: - job: benchmarks-startup artifacts: true - - job: benchmarks-load + - job: benchmarks-load-petclinic + artifacts: true + - job: benchmarks-load-insecure-bank artifacts: true - job: benchmarks-dacapo artifacts: true diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 77ae5be643d..2dd800fa7e5 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -3,22 +3,22 @@ import {checkResponse, isOk, isRedirect} from "../../utils/k6.js"; const variants = { "no_agent": { - "APP_URL": 'http://localhost:8086', + "APP_URL": 'http://localhost:8080', }, "tracing": { - "APP_URL": 'http://localhost:8087', + "APP_URL": 'http://localhost:8081', }, "profiling": { - "APP_URL": 'http://localhost:8088', + "APP_URL": 'http://localhost:8082', }, "iast": { - "APP_URL": 'http://localhost:8089', + "APP_URL": 'http://localhost:8083', }, "iast_GLOBAL": { - "APP_URL": 'http://localhost:8090', + "APP_URL": 'http://localhost:8084', }, "iast_FULL": { - "APP_URL": 'http://localhost:8091', + "APP_URL": 'http://localhost:8085', }, } diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index 92c423cdc4d..4cae95567f2 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -18,11 +18,11 @@ start_server() { echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log [PID=$PID]" } -start_server "no_agent" "-Dserver.port=8086" "taskset -c 48 " & -start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8087" "taskset -c 49 " & -start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8088" "taskset -c 50 " & -start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8089" "taskset -c 51 " & -start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8090" "taskset -c 52 " & -start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8091" "taskset -c 53 " & +start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " & +start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " & +start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " & +start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " & +start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " & +start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " & wait diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index b00e919473a..27cad120029 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -27,9 +27,19 @@ fi source "${UTILS_DIR}/update-java-version.sh" 17 -run_app_benchmark() { - local app=$1 - +# Optional second argument to run a specific app for load benchmarks (i.e. "petclinic" or "insecure-bank") +app_filter=${2:-""} + +for app in *; do + if [[ ! -d "${app}" ]]; then + continue + fi + + # Skip if app filter is specified and doesn't match + if [[ -n "${app_filter}" ]] && [[ "${app}" != "${app_filter}" ]]; then + continue + fi + message "${type} benchmark: ${app} started" export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" @@ -42,15 +52,9 @@ run_app_benchmark() { if [ "${app}" == "petclinic" ]; then HEALTHCHECK_URL=http://localhost:8082 REPETITIONS_COUNT=5 - PORT_START=8080 - PORT_END=8085 - HEALTHCHECK_PATH="" elif [ "${app}" == "insecure-bank" ]; then - HEALTHCHECK_URL=http://localhost:8088/login + HEALTHCHECK_URL=http://localhost:8082/login REPETITIONS_COUNT=5 - PORT_START=8086 - PORT_END=8091 - HEALTHCHECK_PATH="/login" else echo "Unknown app ${app}" exit 1 @@ -60,9 +64,15 @@ run_app_benchmark() { bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & echo "Waiting for serves to start..." - for port in $(seq $PORT_START $PORT_END); do - healthcheck http://localhost:$port$HEALTHCHECK_PATH - done + if [ "${app}" == "petclinic" ]; then + for port in $(seq 8080 8085); do + healthcheck http://localhost:$port + done + elif [ "${app}" == "insecure-bank" ]; then + for port in $(seq 8080 8085); do + healthcheck http://localhost:$port/login + done + fi echo "Servers are up!" ( @@ -72,15 +82,4 @@ run_app_benchmark() { done message "${type} benchmark: ${app} finished" -} - -# Run petclinic and insecure-bank benchmarks in parallel to reduce total runtime -for app in *; do - if [[ ! -d "${app}" ]]; then - continue - fi - run_app_benchmark "${app}" & done - -# Wait for all background jobs to complete -wait From bbd98cece1652543701c9366943b5feea686ca2a Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Fri, 7 Nov 2025 11:13:51 -0500 Subject: [PATCH 5/7] Add missing load benchmark split --- .gitlab/macrobenchmarks.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/macrobenchmarks.yml b/.gitlab/macrobenchmarks.yml index d2ad417c2dd..2cab05cb9ea 100644 --- a/.gitlab/macrobenchmarks.yml +++ b/.gitlab/macrobenchmarks.yml @@ -93,7 +93,9 @@ check-slo-breaches: artifacts: true - job: benchmarks-startup artifacts: true - - job: benchmarks-load + - job: benchmarks-load-petclinic + artifacts: true + - job: benchmarks-load-insecure-bank artifacts: true - job: benchmarks-dacapo artifacts: true From 0bdbb63a8f627420aa562c972320b8d82fec03ad Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Fri, 7 Nov 2025 12:42:01 -0500 Subject: [PATCH 6/7] Empty commit to run CI benchmarks again From 9e108de086f0a029f8b1ffb2463e3b891a28d42a Mon Sep 17 00:00:00 2001 From: Sarah Chen Date: Mon, 10 Nov 2025 16:50:16 -0500 Subject: [PATCH 7/7] Run load benchmarks 5 times in parallel --- .gitlab/benchmarks.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 11ec5c6b6ac..bdf0d807112 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -37,6 +37,9 @@ benchmarks-startup: benchmarks-load-petclinic: extends: .benchmarks + parallel: + matrix: + - RUN_ID: [1, 2, 3, 4, 5] script: - !reference [ .benchmarks, script ] - ./steps/capture-hardware-software-info.sh @@ -45,6 +48,9 @@ benchmarks-load-petclinic: benchmarks-load-insecure-bank: extends: .benchmarks + parallel: + matrix: + - RUN_ID: [1, 2, 3, 4, 5] script: - !reference [ .benchmarks, script ] - ./steps/capture-hardware-software-info.sh @@ -70,8 +76,14 @@ benchmarks-post-results: - job: benchmarks-startup artifacts: true - job: benchmarks-load-petclinic + parallel: + matrix: + - RUN_ID: [1, 2, 3, 4, 5] artifacts: true - job: benchmarks-load-insecure-bank + parallel: + matrix: + - RUN_ID: [1, 2, 3, 4, 5] artifacts: true - job: benchmarks-dacapo artifacts: true