Setting read only during shutdown

Summary:
Setting read only = 1 during shutdown after attempting to kill
all connections but before shutting down plugins. This way we can be
sure that no writes are being processed when semi-sync/raft plugins are
shutdown.

Reviewed By: hermanlee

Differential Revision: D27539886

Author: abhinav04sharma

mysql-test/r/mysqld--help-notwin.result

@@ -2293,6 +2293,10 @@ The following options may be given as the first argument:
  characteristics (isolation level, read only/read write,
  snapshot - but not any work done / data modified within
  the transaction).
+ --set-read-only-on-shutdown 
+ Set read_only and super_read_only in shutdown path after
+ trying to kill connections but before shutting down
+ plugins
  --sha256-password-proxy-users 
  If set to FALSE (the default), then the sha256_password
  authentication plugin will not signal for authenticated
@@ -3390,6 +3394,7 @@ session-track-schema TRUE
 session-track-state-change FALSE
 session-track-system-variables time_zone,autocommit,character_set_client,character_set_results,character_set_connection
 session-track-transaction-info OFF
+set-read-only-on-shutdown FALSE
 sha256-password-proxy-users FALSE
 show-create-table-verbosity FALSE
 show-gipk-in-create-table-and-information-schema TRUE

mysql-test/r/read_only_on_shutdown.result

@@ -0,0 +1,20 @@
+SET GLOBAL set_read_only_on_shutdown = 1;
+SET GLOBAL DEBUG = "+d,after_shutdown_read_only";
+CREATE TABLE t1 (pk INT, PRIMARY KEY (pk)) ENGINE = INNODB;
+"Start the sql client and run shutdown sql command"
+"Reached the point just after setting read_only in the shutdown path"
+SET DEBUG_SYNC = "now wait_for signal.reached";
+"Try to execute a write"
+select @@global.read_only, @@global.super_read_only;
+@@global.read_only	@@global.super_read_only
+1	1
+INSERT INTO t1 VALUES(1);
+ERROR HY000: The MySQL server is running with the --super-read-only option so it cannot execute this statement
+"Continuing the shutdown"
+SET GLOBAL DEBUG = "-d,after_shutdown_read_only";
+SET DEBUG_SYNC = "now signal signal.done";
+"Starting the server again"
+"Table contents (should be empty)"
+SELECT * FROM t1;
+pk
+DROP TABLE t1;

mysql-test/suite/sys_vars/r/set_read_only_on_shutdown_basic.result

@@ -0,0 +1,7 @@
+Default value of set_read_only_on_shutdown is 0
+SELECT @@global.set_read_only_on_shutdown;
+@@global.set_read_only_on_shutdown
+0
+SELECT @@session.set_read_only_on_shutdown;
+ERROR HY000: Variable 'set_read_only_on_shutdown' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'

mysql-test/suite/sys_vars/t/set_read_only_on_shutdown_basic.test

@@ -0,0 +1,14 @@
+-- source include/load_sysvars.inc
+
+####
+# Verify default value 0
+####
+--echo Default value of set_read_only_on_shutdown is 0
+SELECT @@global.set_read_only_on_shutdown;
+
+####
+# Verify that this is not a session variable #
+####
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@session.set_read_only_on_shutdown;
+--echo Expected error 'Variable is a GLOBAL variable'

mysql-test/t/read_only_on_shutdown.test

@@ -0,0 +1,33 @@
+source include/have_debug_sync.inc;
+
+SET GLOBAL set_read_only_on_shutdown = 1;
+SET GLOBAL DEBUG = "+d,after_shutdown_read_only";
+
+CREATE TABLE t1 (pk INT, PRIMARY KEY (pk)) ENGINE = INNODB;
+
+echo "Start the sql client and run shutdown sql command";
+exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
+exec $MYSQL -e "shutdown;" 2>&1;
+
+echo "Reached the point just after setting read_only in the shutdown path";
+SET DEBUG_SYNC = "now wait_for signal.reached";
+
+echo "Try to execute a write";
+select @@global.read_only, @@global.super_read_only;
+error 1290;
+INSERT INTO t1 VALUES(1);
+
+echo "Continuing the shutdown";
+SET GLOBAL DEBUG = "-d,after_shutdown_read_only";
+SET DEBUG_SYNC = "now signal signal.done";
+source include/wait_until_disconnected.inc;
+
+echo "Starting the server again";
+exec echo "restart:" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
+enable_reconnect;
+source include/wait_until_connected_again.inc;
+
+echo "Table contents (should be empty)";
+SELECT * FROM t1;
+
+DROP TABLE t1;

sql/mysqld.cc

@@ -1335,6 +1335,9 @@ uint default_password_lifetime = 0;
 bool password_require_current = false;
 std::atomic<bool> partial_revokes;
 bool opt_partial_revokes;  // Initialized through Sys_var
+bool set_read_only_on_shutdown = false;
+extern bool fix_read_only(sys_var *self, THD *thd, enum_var_type type);
+extern bool fix_super_read_only(sys_var *self, THD *thd, enum_var_type type);
 
 mysql_mutex_t LOCK_default_password_lifetime;
 mysql_mutex_t LOCK_mandatory_roles;
@@ -2587,6 +2590,35 @@ static void close_connections(void) {
 
   (void)RUN_HOOK(server_state, before_server_shutdown, (nullptr));
 
+  // case: block all writes by setting read_only=1 and super_read_only=1
+  if (set_read_only_on_shutdown) {
+    THD *thd = new THD;
+    thd->thread_stack = reinterpret_cast<char *>(&(thd));
+    thd->store_globals();
+
+    // NO_LINT_DEBUG
+    sql_print_information("Setting read_only=1 during shutdown");
+    mysql_mutex_lock(&LOCK_global_system_variables);
+    read_only = super_read_only = true;
+    if (fix_read_only(nullptr, thd, OPT_GLOBAL) ||
+        fix_super_read_only(nullptr, thd, OPT_GLOBAL))
+      // NO_LINT_DEBUG
+      sql_print_error("Setting read_only=1 failed during shutdown");
+    else
+      // NO_LINT_DEBUG
+      sql_print_information("Successfully set read_only=1 during shutdown");
+    mysql_mutex_unlock(&LOCK_global_system_variables);
+
+    DBUG_EXECUTE_IF("after_shutdown_read_only", {
+      const char act[] = "now signal signal.reached wait_for signal.done";
+      assert(opt_debug_sync_timeout > 0);
+      assert(!debug_sync_set_action(thd, STRING_WITH_LEN(act)));
+    });
+
+    thd->restore_globals();
+    delete thd;
+  }
+
   Per_thread_connection_handler::kill_blocked_pthreads();
 
   uint dump_thread_count = 0;

sql/mysqld.h

@@ -478,6 +478,7 @@ extern uint net_compression_level;
 extern long zstd_net_compression_level;
 extern long lz4f_net_compression_level;
 extern bool enable_blind_replace;
+extern bool set_read_only_on_shutdown;
 
 /* SHOW STATS var: Name of current timer */
 extern const char *timer_in_use;

sql/sys_vars.cc

@@ -4046,7 +4046,7 @@ static void event_scheduler_restart(THD *thd) {
   }
 }
 
-static bool fix_read_only(sys_var *self, THD *thd, enum_var_type) {
+bool fix_read_only(sys_var *self, THD *thd, enum_var_type) {
   bool result = true;
   bool new_read_only = read_only;  // make a copy before releasing a mutex
   DBUG_TRACE;
@@ -4139,7 +4139,7 @@ static bool fix_read_only(sys_var *self, THD *thd, enum_var_type) {
   return result;
 }
 
-static bool fix_super_read_only(sys_var *, THD *thd, enum_var_type type) {
+bool fix_super_read_only(sys_var *, THD *thd, enum_var_type type) {
   DBUG_TRACE;
 
   /* return if no changes: */
@@ -9511,3 +9511,9 @@ static Sys_var_enum Sys_sql_duplicate_executions_control(
     GLOBAL_VAR(sql_duplicate_executions_control), CMD_LINE(OPT_ARG),
     control_level_values, DEFAULT(CONTROL_LEVEL_OFF), NO_MUTEX_GUARD,
     NOT_IN_BINLOG, ON_CHECK(NULL), ON_UPDATE(NULL));
+
+static Sys_var_bool Sys_set_read_only_on_shutdown(
+    "set_read_only_on_shutdown",
+    "Set read_only and super_read_only in shutdown path after trying to "
+    "kill connections but before shutting down plugins",
+    GLOBAL_VAR(set_read_only_on_shutdown), CMD_LINE(OPT_ARG), DEFAULT(false));