add toxiproxy option for localtests (#1591)

Author: meiji163

doc/local-tests.md

@@ -34,3 +34,11 @@ TEST_MYSQL_IMAGE="mysql-server:8.0.16" ./script/docker-gh-ost-replica-tests up
 # cleanup containers
 ./script/docker-gh-ost-replica-tests down
 ```
+
+Pass the `-t` flag to run the tests with a toxiproxy between gh-ost and the MySQL replica. This simulates network conditions where MySQL connections are closed unexpectedly.
+
+```shell
+# run tests with toxiproxy
+./script/docker-gh-ost-replica-tests up -t
+./script/docker-gh-ost-replica-tests run -t
+```

go/cmd/gh-ost/main.go

@@ -143,6 +143,7 @@ func main() {
 	flag.BoolVar(&migrationContext.IncludeTriggers, "include-triggers", false, "When true, the triggers (if exist) will be created on the new table")
 	flag.StringVar(&migrationContext.TriggerSuffix, "trigger-suffix", "", "Add a suffix to the trigger name (i.e '_v2'). Requires '--include-triggers'")
 	flag.BoolVar(&migrationContext.RemoveTriggerSuffix, "remove-trigger-suffix-if-exists", false, "Remove given suffix from name of trigger. Requires '--include-triggers' and '--trigger-suffix'")
+	flag.BoolVar(&migrationContext.SkipPortValidation, "skip-port-validation", false, "Skip port validation for MySQL connections")
 
 	maxLoad := flag.String("max-load", "", "Comma delimited status-name=threshold. e.g: 'Threads_running=100,Threads_connected=500'. When status exceeds threshold, app throttles writes")
 	criticalLoad := flag.String("critical-load", "", "Comma delimited status-name=threshold, same format as --max-load. When status exceeds threshold, app panics and quits")

localtests/test.sh

@@ -12,6 +12,7 @@ test_logfile=/tmp/gh-ost-test.log
 default_ghost_binary=/tmp/gh-ost-test
 ghost_binary=""
 docker=false
+toxiproxy=false
 storage_engine=innodb
 exec_command_file=/tmp/gh-ost-test.bash
 ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql
@@ -29,14 +30,17 @@ original_sql_mode=
 sysbench_pid=
 
 OPTIND=1
-while getopts "b:s:d" OPTION; do
+while getopts "b:s:dt" OPTION; do
     case $OPTION in
     b)
         ghost_binary="$OPTARG"
         ;;
     s)
         storage_engine="$OPTARG"
         ;;
+    t)
+        toxiproxy=true
+        ;;
     d)
         docker=true
         ;;
@@ -75,6 +79,20 @@ verify_master_and_replica() {
     read replica_host replica_port <<<$(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
     [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1"
     echo "# replica verified at $replica_host:$replica_port"
+
+    if [ "$docker" = true ]; then
+        master_host="0.0.0.0"
+        master_port="3307"
+        echo "# using docker master at $master_host:$master_port"
+        replica_host="0.0.0.0"
+        if [ "$toxiproxy" = true ]; then
+            replica_port="23308"
+            echo "# using toxiproxy replica at $replica_host:$replica_port"
+        else
+            replica_port="3308"
+            echo "# using docker replica at $replica_host:$replica_port"
+        fi
+    fi
 }
 
 exec_cmd() {
@@ -154,13 +172,6 @@ test_single() {
     local test_name
     test_name="$1"
 
-    if [ "$docker" = true ]; then
-        master_host="0.0.0.0"
-        master_port="3307"
-        replica_host="0.0.0.0"
-        replica_port="3308"
-    fi
-
     if [ -f $tests_path/$test_name/ignore_versions ]; then
         ignore_versions=$(cat $tests_path/$test_name/ignore_versions)
         mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version")
@@ -199,6 +210,9 @@ test_single() {
     if [ -f $tests_path/$test_name/extra_args ]; then
         extra_args=$(cat $tests_path/$test_name/extra_args)
     fi
+    if [ "$toxiproxy" = true ]; then
+        extra_args+=" --skip-port-validation"
+    fi
     orig_columns="*"
     ghost_columns="*"
     order_by=""

script/docker-gh-ost-replica-tests

@@ -5,11 +5,15 @@
 # Set the environment var TEST_MYSQL_IMAGE to change the docker image.
 #
 # Usage:
-#   docker-gh-ost-replica-tests up      start the containers
-#   docker-gh-ost-replica-tests down    remove the containers
-#   docker-gh-ost-replica-tests run     run replica tests on the containers
+#   docker-gh-ost-replica-tests up [-t]     start the containers
+#   docker-gh-ost-replica-tests down        remove the containers
+#   docker-gh-ost-replica-tests run [-t]    run replica tests on the containers
+#
+# Flags:
+#   -t      use a toxiproxy for replica connection to simulate dropped connections
 
 set -e
+toxiproxy=false
 
 GH_OST_ROOT=$(git rev-parse --show-toplevel)
 if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then
@@ -47,6 +51,22 @@ mysql-replica() {
     fi
 }
 
+create_toxiproxy() {
+    curl --fail -X POST http://localhost:8474/proxies \
+        -H "Content-Type: application/json" \
+        -d '{"name": "mysql_proxy",
+            "listen": "0.0.0.0:23308",
+            "upstream": "host.docker.internal:3308"}'
+    echo
+
+    curl --fail -X POST http://localhost:8474/proxies/mysql_proxy/toxics \
+        -H "Content-Type: application/json" \
+        -d '{"name": "limit_data_downstream",
+            "type": "limit_data",
+            "attributes": {"bytes": 300000}}'
+    echo
+}
+
 setup() {
     [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41"
 
@@ -59,6 +79,22 @@ setup() {
         MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON"
     fi
     (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_SHA2_RSA_KEYS_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" envsubst <"$compose_file") >"$compose_file.tmp"
+
+    if [ "$toxiproxy" = true ]; then
+        echo "Starting toxiproxy container..."
+        cat <<EOF >>"$compose_file.tmp"
+  mysql-toxiproxy:
+    image: "ghcr.io/shopify/toxiproxy:latest"
+    container_name: mysql-toxiproxy
+    ports:
+      - '8474:8474'
+      - '23308:23308'
+    expose:
+      - '23308'
+      - '8474'
+EOF
+    fi
+
     docker compose -f "$compose_file.tmp" up -d --wait
 
     echo "Waiting for MySQL..."
@@ -80,23 +116,36 @@ setup() {
         mysql-replica -e "start slave;"
     fi
     echo "OK"
+
+    if [ "$toxiproxy" = true ]; then
+        echo "Creating toxiproxy..."
+        create_toxiproxy
+        echo "OK"
+    fi
 }
 
 teardown() {
     echo "Stopping containers..."
     docker stop mysql-replica
     docker stop mysql-primary
+    docker stop mysql-toxiproxy 2>/dev/null || true
     echo "Removing containers..."
     docker rm mysql-replica
     docker rm mysql-primary
+    docker rm mysql-toxiproxy 2>/dev/null || true
 }
 
 main() {
-    if [[ "$1" == "up" ]]; then
+    local cmd="$1"
+    local tflag=
+    if [[ "$2" == "-t" ]]; then
+        toxiproxy=true
+    fi
+    if [[ "$cmd" == "up" ]]; then
         setup
-    elif [[ "$1" == "down" ]]; then
+    elif [[ "$cmd" == "down" ]]; then
         teardown
-    elif [[ "$1" == "run" ]]; then
+    elif [[ "$cmd" == "run" ]]; then
         shift 1
         "$GH_OST_ROOT/localtests/test.sh" -d "$@"
     fi