Parallelize into separate jobs

Author: sarahchen6

.gitlab/benchmarks.yml

@@ -35,12 +35,20 @@ benchmarks-startup:
     - ./steps/run-benchmarks.sh startup
     - ./steps/analyze-results.sh startup
 
-benchmarks-load:
+benchmarks-load-petclinic:
   extends: .benchmarks
   script:
     - !reference [ .benchmarks, script ]
     - ./steps/capture-hardware-software-info.sh
-    - ./steps/run-benchmarks.sh load
+    - ./steps/run-benchmarks.sh load petclinic
+    - ./steps/analyze-results.sh load
+
+benchmarks-load-insecure-bank:
+  extends: .benchmarks
+  script:
+    - !reference [ .benchmarks, script ]
+    - ./steps/capture-hardware-software-info.sh
+    - ./steps/run-benchmarks.sh load insecure-bank
     - ./steps/analyze-results.sh load
 
 benchmarks-dacapo:
@@ -61,7 +69,9 @@ benchmarks-post-results:
   needs:
     - job: benchmarks-startup
       artifacts: true
-    - job: benchmarks-load
+    - job: benchmarks-load-petclinic
+      artifacts: true
+    - job: benchmarks-load-insecure-bank
       artifacts: true
     - job: benchmarks-dacapo
       artifacts: true

benchmark/load/insecure-bank/k6.js

@@ -3,22 +3,22 @@ import {checkResponse, isOk, isRedirect} from "../../utils/k6.js";
 
 const variants = {
   "no_agent": {
-    "APP_URL": 'http://localhost:8086',
+    "APP_URL": 'http://localhost:8080',
   },
   "tracing": {
-    "APP_URL": 'http://localhost:8087',
+    "APP_URL": 'http://localhost:8081',
   },
   "profiling": {
-    "APP_URL": 'http://localhost:8088',
+    "APP_URL": 'http://localhost:8082',
   },
   "iast": {
-    "APP_URL": 'http://localhost:8089',
+    "APP_URL": 'http://localhost:8083',
   },
   "iast_GLOBAL": {
-    "APP_URL": 'http://localhost:8090',
+    "APP_URL": 'http://localhost:8084',
   },
   "iast_FULL": {
-    "APP_URL": 'http://localhost:8091',
+    "APP_URL": 'http://localhost:8085',
   },
 }
 

benchmark/load/insecure-bank/start-servers.sh

@@ -18,11 +18,11 @@ start_server() {
   echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log [PID=$PID]"
 }
 
-start_server "no_agent" "-Dserver.port=8086" "taskset -c 48 " &
-start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8087" "taskset -c 49 " &
-start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8088" "taskset -c 50 " &
-start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8089" "taskset -c 51 " &
-start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8090" "taskset -c 52 " &
-start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8091" "taskset -c 53 " &
+start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " &
+start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " &
+start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " &
+start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " &
+start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " &
+start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " &
 
 wait

benchmark/load/run.sh

@@ -27,9 +27,19 @@ fi
 
 source "${UTILS_DIR}/update-java-version.sh" 17
 
-run_app_benchmark() {
-  local app=$1
-  
+# Optional second argument to run a specific app for load benchmarks (i.e. "petclinic" or "insecure-bank")
+app_filter=${2:-""}
+
+for app in *; do
+  if [[ ! -d "${app}" ]]; then
+    continue
+  fi
+
+  # Skip if app filter is specified and doesn't match
+  if [[ -n "${app_filter}" ]] && [[ "${app}" != "${app_filter}" ]]; then
+    continue
+  fi
+
   message "${type} benchmark: ${app} started"
 
   export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}"
@@ -42,15 +52,9 @@ run_app_benchmark() {
   if [ "${app}" == "petclinic" ]; then
     HEALTHCHECK_URL=http://localhost:8082
     REPETITIONS_COUNT=5
-    PORT_START=8080
-    PORT_END=8085
-    HEALTHCHECK_PATH=""
   elif [ "${app}" == "insecure-bank" ]; then
-    HEALTHCHECK_URL=http://localhost:8088/login
+    HEALTHCHECK_URL=http://localhost:8082/login
     REPETITIONS_COUNT=5
-    PORT_START=8086
-    PORT_END=8091
-    HEALTHCHECK_PATH="/login"
   else
     echo "Unknown app ${app}"
     exit 1
@@ -60,9 +64,15 @@ run_app_benchmark() {
     bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" &
 
     echo "Waiting for serves to start..."
-    for port in $(seq $PORT_START $PORT_END); do
-      healthcheck http://localhost:$port$HEALTHCHECK_PATH
-    done
+    if [ "${app}" == "petclinic" ]; then
+      for port in $(seq 8080 8085); do
+        healthcheck http://localhost:$port
+      done
+    elif [ "${app}" == "insecure-bank" ]; then
+      for port in $(seq 8080 8085); do
+        healthcheck http://localhost:$port/login
+      done
+    fi
     echo "Servers are up!"
 
     (
@@ -72,15 +82,4 @@ run_app_benchmark() {
   done
 
   message "${type} benchmark: ${app} finished"
-}
-
-# Run petclinic and insecure-bank benchmarks in parallel to reduce total runtime
-for app in *; do
-  if [[ ! -d "${app}" ]]; then
-    continue
-  fi
-  run_app_benchmark "${app}" &
 done
-
-# Wait for all background jobs to complete
-wait