From 854f8686b729aa6afa20006799ac862dbfa834e0 Mon Sep 17 00:00:00 2001 From: Mark Callaghan Date: Thu, 11 May 2023 19:52:50 +0000 Subject: [PATCH] Fix issues 1200 and 1295, add rocksdb_compact_lzero_now global variable This fixes issue 1295 and has a workaround for issue 1200. https://github.com/facebook/mysql-5.6/issues/1200 https://github.com/facebook/mysql-5.6/issues/1295 The fix for issue 1295 is to get the value of the base level from RocksDB rather than assuming that L0 compacts into L1 because L1 might not be the base level when dynamic leveled compaction is used -- in that case the base level can be Ln where n > 1 because L1 isn't needed yet. There are two workarounds for issue 1200. The hacky one is to sleep for 1 second between requesting memtable flush and L0 -> base_level compaction. The alternative is to add a new global variable, rocksdb_compact_lzero_now, that when set will request L0 -> base_level compaction. This allows a client to first set rocksdb_force_flush_memtable_now, wait for that to finish, then set rocksdb_compact_lzero_now. Other changes: * rocksdb_force_flush_memtable_now default value changed to OFF to match what is done for rocksdb_force_flush_memtable_and_lzero_now and rocksdb_compact_lzero_now * confirm that the value to which these variables are set can be parsed, that wasn't done for all of them * don't raise an error when these are set to OFF, that will be a no-op Note: these variables are triggers, an action is taken when this is done: set global var = ON | true | 1 But these variables always show the value 0 (OFF, false). Their value doesn't change. --- mysql-test/suite/rocksdb/r/rocksdb.result | 3 +- .../r/rocksdb_compact_lzero_now_basic.result | 83 +++++++++++++ ..._flush_memtable_and_lzero_now_basic.result | 39 +++++- ...ksdb_force_flush_memtable_now_basic.result | 54 +++++---- .../t/rocksdb_compact_lzero_now_basic.test | 20 ++++ ...ce_flush_memtable_and_lzero_now_basic.test | 5 +- ...ocksdb_force_flush_memtable_now_basic.test | 6 +- storage/rocksdb/ha_rocksdb.cc | 111 +++++++++++++++--- 8 files changed, 275 insertions(+), 46 deletions(-) create mode 100644 mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result create mode 100644 mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test diff --git a/mysql-test/suite/rocksdb/r/rocksdb.result b/mysql-test/suite/rocksdb/r/rocksdb.result index 9d67a82e9bb4..b123ede438ee 100644 --- a/mysql-test/suite/rocksdb/r/rocksdb.result +++ b/mysql-test/suite/rocksdb/r/rocksdb.result @@ -936,6 +936,7 @@ rocksdb_column_default_value_as_expression ON rocksdb_commit_in_the_middle OFF rocksdb_commit_time_batch_for_recovery ON rocksdb_compact_cf +rocksdb_compact_lzero_now OFF rocksdb_compaction_readahead_size 0 rocksdb_compaction_sequential_deletes 0 rocksdb_compaction_sequential_deletes_count_sd OFF @@ -984,7 +985,7 @@ rocksdb_flush_log_at_trx_commit 1 rocksdb_force_compute_memtable_stats ON rocksdb_force_compute_memtable_stats_cachetime 0 rocksdb_force_flush_memtable_and_lzero_now OFF -rocksdb_force_flush_memtable_now ON +rocksdb_force_flush_memtable_now OFF rocksdb_force_index_records_in_range 0 rocksdb_ignore_unknown_options ON rocksdb_index_type kBinarySearch diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result new file mode 100644 index 000000000000..28302a0a4671 --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_compact_lzero_now_basic.result @@ -0,0 +1,83 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +SET @start_global_value = @@global.ROCKSDB_COMPACT_LZERO_NOW; +SELECT @start_global_value; +@start_global_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to 1" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = 1; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to on" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = on; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to true" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = true; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to 0" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = 0; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to off" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = off; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_COMPACT_LZERO_NOW to false" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = false; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +"Trying to set variable @@session.ROCKSDB_COMPACT_LZERO_NOW to 444. It should fail because it is not session." +SET @@session.ROCKSDB_COMPACT_LZERO_NOW = 444; +ERROR HY000: Variable 'rocksdb_compact_lzero_now' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' +SET @@global.ROCKSDB_COMPACT_LZERO_NOW = @start_global_value; +SELECT @@global.ROCKSDB_COMPACT_LZERO_NOW; +@@global.ROCKSDB_COMPACT_LZERO_NOW +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result index 68cfeb07fc71..a3e506753d90 100644 --- a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result @@ -1,7 +1,10 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); -INSERT INTO valid_values VALUES(0); INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; SET @start_global_value = @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; SELECT @start_global_value; @@ -18,6 +21,26 @@ SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW 0 +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to on" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = on; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to true" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = true; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to 0" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = 0; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@ -28,8 +51,18 @@ SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW 0 -"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to on" -SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = on; +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to off" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = off; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW +0 +"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to false" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = false; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW 0 diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result index 53665a7cedaa..59f6c832da39 100644 --- a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_now_basic.result @@ -2,70 +2,82 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); INSERT INTO valid_values VALUES('on'); INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO invalid_values VALUES(0); -INSERT INTO invalid_values VALUES('off'); -INSERT INTO invalid_values VALUES('false'); SET @start_global_value = @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; SELECT @start_global_value; @start_global_value -1 +0 '# Setting to valid values in global scope#' "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 1" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 1; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Setting the global scope variable back to default" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to on" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = on; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Setting the global scope variable back to default" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to true" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = true; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 "Setting the global scope variable back to default" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 -"Trying to set variable @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 444. It should fail because it is not session." -SET @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 444; -ERROR HY000: Variable 'rocksdb_force_flush_memtable_now' is a GLOBAL variable and should be set with SET GLOBAL -'# Testing with invalid values in global scope #' +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 0" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 0; -Got one of the listed errors SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to off" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = off; -Got one of the listed errors SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW +0 "Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to false" SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = false; -Got one of the listed errors SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = DEFAULT; +SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; +@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW +0 +"Trying to set variable @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW to 444. It should fail because it is not session." +SET @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = 444; +ERROR HY000: Variable 'rocksdb_force_flush_memtable_now' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW = @start_global_value; SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW; @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW -1 +0 DROP TABLE valid_values; DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test new file mode 100644 index 000000000000..eb9431a7d09d --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_compact_lzero_now_basic.test @@ -0,0 +1,20 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; + +--let $sys_var=ROCKSDB_COMPACT_LZERO_NOW +--let $read_only=0 +--let $session=0 +--let $sticky=1 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test index 5eeac5010407..39bbe0eb6e4d 100644 --- a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test @@ -2,8 +2,11 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); -INSERT INTO valid_values VALUES(0); INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test index b493d5a80eef..4761f9b44406 100644 --- a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_now_basic.test @@ -4,11 +4,11 @@ CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; INSERT INTO valid_values VALUES(1); INSERT INTO valid_values VALUES('on'); INSERT INTO valid_values VALUES('true'); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('off'); +INSERT INTO valid_values VALUES('false'); CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO invalid_values VALUES(0); -INSERT INTO invalid_values VALUES('off'); -INSERT INTO invalid_values VALUES('false'); --let $sys_var=ROCKSDB_FORCE_FLUSH_MEMTABLE_NOW --let $read_only=0 diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 4ab62e81f938..2025e1040a4c 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -542,6 +542,8 @@ static bool parse_fault_injection_params(bool *retryable, uint32_t *failure_ratio, std::vector *types); +static int delete_range(const std::unordered_set &indices); + static void rocksdb_force_flush_memtable_now_stub( THD *const thd MY_ATTRIBUTE((__unused__)), struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), @@ -554,8 +556,11 @@ static int rocksdb_force_flush_memtable_now( void *const var_ptr MY_ATTRIBUTE((__unused__)), struct st_mysql_value *const value MY_ATTRIBUTE((__unused__))) { bool parsed_value = false; - if (mysql_value_to_bool(value, &parsed_value) != 0 || !parsed_value) { + if (mysql_value_to_bool(value, &parsed_value) != 0) { return 1; + } else if (!parsed_value) { + // Setting to OFF is a no-op and this supports mtr tests + return HA_EXIT_SUCCESS; } // NO_LINT_DEBUG @@ -564,22 +569,11 @@ static int rocksdb_force_flush_memtable_now( rocksdb_flush_all_memtables(); return HA_EXIT_SUCCESS; } -static int delete_range(const std::unordered_set &indices); -static void rocksdb_force_flush_memtable_and_lzero_now_stub( - THD *const thd MY_ATTRIBUTE((__unused__)), - struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), - void *const var_ptr MY_ATTRIBUTE((__unused__)), - const void *const save MY_ATTRIBUTE((__unused__))) {} -static int rocksdb_force_flush_memtable_and_lzero_now( - THD *const thd MY_ATTRIBUTE((__unused__)), - struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), - void *const var_ptr MY_ATTRIBUTE((__unused__)), - struct st_mysql_value *const value MY_ATTRIBUTE((__unused__))) { +static int rocksdb_compact_lzero() { // NO_LINT_DEBUG LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, - "RocksDB: Manual memtable and L0 flush."); - rocksdb_flush_all_memtables(); + "RocksDB: compact L0."); const Rdb_cf_manager &cf_manager = rdb_get_cf_manager(); rocksdb::CompactionOptions c_options = rocksdb::CompactionOptions(); @@ -594,6 +588,25 @@ static int rocksdb_force_flush_memtable_and_lzero_now( cf_handle->GetDescriptor(&cf_descr); c_options.output_file_size_limit = cf_descr.options.target_file_size_base; + // Lets RocksDB use the configured compression for this level + c_options.compression = rocksdb::kDisableCompressionOption; + + uint64_t base_level; + if (!rdb->GetIntProperty(cf_handle.get(), + rocksdb::DB::Properties::kBaseLevel, &base_level)) { + LogPluginErrMsg( + ERROR_LEVEL, ER_LOG_PRINTF_MSG, + "MyRocks: compact L0 cannot get base level"); + break; + } + + if (base_level == 0) { + LogPluginErrMsg( + ERROR_LEVEL, ER_LOG_PRINTF_MSG, + "MyRocks: compact L0 cannot flush to base level when 0"); + break; + } + assert(metadata.levels[0].level == 0); std::vector file_names; for (auto &file : metadata.levels[0].files) { @@ -601,11 +614,14 @@ static int rocksdb_force_flush_memtable_and_lzero_now( } if (file_names.empty()) { + LogPluginErrMsg( + INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, + "MyRocks: no files in L0"); break; } rocksdb::Status s; - s = rdb->CompactFiles(c_options, cf_handle.get(), file_names, 1); + s = rdb->CompactFiles(c_options, cf_handle.get(), file_names, base_level); if (!s.ok()) { std::shared_ptr cfh = @@ -643,6 +659,60 @@ static int rocksdb_force_flush_memtable_and_lzero_now( return num_errors == 0 ? HA_EXIT_SUCCESS : HA_EXIT_FAILURE; } +static void rocksdb_compact_lzero_now_stub( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + const void *const save MY_ATTRIBUTE((__unused__))) {} + +static int rocksdb_compact_lzero_now( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + struct st_mysql_value *const value) { + bool parsed_value = false; + + if (mysql_value_to_bool(value, &parsed_value) != 0) { + return 1; + } else if (!parsed_value) { + // Setting to OFF is a no-op and this supports mtr tests + return HA_EXIT_SUCCESS; + } + + return rocksdb_compact_lzero(); +} + +static void rocksdb_force_flush_memtable_and_lzero_now_stub( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + const void *const save MY_ATTRIBUTE((__unused__))) {} + +static int rocksdb_force_flush_memtable_and_lzero_now( + THD *const thd MY_ATTRIBUTE((__unused__)), + struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), + void *const var_ptr MY_ATTRIBUTE((__unused__)), + struct st_mysql_value *const value) { + bool parsed_value = false; + + if (mysql_value_to_bool(value, &parsed_value) != 0) { + return 1; + } else if (!parsed_value) { + // Setting to OFF is a no-op and this supports mtr tests + return HA_EXIT_SUCCESS; + } + + // NO_LINT_DEBUG + LogPluginErrMsg(INFORMATION_LEVEL, ER_LOG_PRINTF_MSG, + "RocksDB: Manual memtable and L0 flush."); + rocksdb_flush_all_memtables(); + + // Try to avoid https://github.com/facebook/mysql-5.6/issues/1200 + my_sleep(1000000); + + return rocksdb_compact_lzero(); +} + static void rocksdb_cancel_manual_compactions_stub( THD *const thd MY_ATTRIBUTE((__unused__)), struct SYS_VAR *const var MY_ATTRIBUTE((__unused__)), @@ -825,8 +895,9 @@ static bool rocksdb_ignore_unknown_options = 1; static bool rocksdb_enable_2pc = 0; static char *rocksdb_strict_collation_exceptions; static bool rocksdb_collect_sst_properties = 1; -static bool rocksdb_force_flush_memtable_now_var = 1; +static bool rocksdb_force_flush_memtable_now_var = 0; static bool rocksdb_force_flush_memtable_and_lzero_now_var = 0; +static bool rocksdb_compact_lzero_now_var = 0; static bool rocksdb_cancel_manual_compactions_var = 0; static bool rocksdb_enable_ttl = 1; static bool rocksdb_enable_ttl_read_filtering = 1; @@ -2441,7 +2512,12 @@ static MYSQL_SYSVAR_BOOL( PLUGIN_VAR_RQCMDARG, "Forces memstore flush which may block all write requests so be careful", rocksdb_force_flush_memtable_now, rocksdb_force_flush_memtable_now_stub, - true); + false); + +static MYSQL_SYSVAR_BOOL( + compact_lzero_now, rocksdb_compact_lzero_now_var, + PLUGIN_VAR_RQCMDARG, "Compacts all L0 files.", + rocksdb_compact_lzero_now, rocksdb_compact_lzero_now_stub, false); static MYSQL_SYSVAR_BOOL( force_flush_memtable_and_lzero_now, @@ -2980,6 +3056,7 @@ static struct SYS_VAR *rocksdb_system_variables[] = { MYSQL_SYSVAR(collect_sst_properties), MYSQL_SYSVAR(force_flush_memtable_now), MYSQL_SYSVAR(force_flush_memtable_and_lzero_now), + MYSQL_SYSVAR(compact_lzero_now), MYSQL_SYSVAR(cancel_manual_compactions), MYSQL_SYSVAR(enable_ttl), MYSQL_SYSVAR(enable_ttl_read_filtering),