Return stats from storage engine directly without table open

Summary:
In 8.0 query from i_s.tables are *much* slower (~5x) most likely due to table open/close are much slower - profiling data showed that 74% time spent on opening tables and 19.71% time spent on reading histograms, only 2.41% spent on the actual stats inside MyRocks.

8.0 provides a new callback get_table_statistics and get_index_column_cardinality that when "overridden", signal MySQL to go call those functions to fill in stats directly, which should be much faster, with some caveats:
* Our stats code path need to work without table open - this fix updates our stat code path to strictly use Rdb_tbl_def and Rdb_key_def.

One catch is that we won't be able to update the auto_inc value correctly because we won't know if the table actually has AUTO_INCREMENT without opening it, so there are some corner cases the value might be NULL (such as immediately after creating a table with AUTO_INCREMENT). This actually aligns with ROCKSDB_DDL table behavior and InnoDB behavior as well. Once you open the table, the cached Rdb_tbl_def::m_auto_inc_val should be updated properly.

* We also need to make sure the lifetime / locking works properly with regarding to DDL and special cases like drop cf.

Fortunately the table_stats code path does take MDL_EXPLICIT lock with the db_name.table_name, so correctness should be straight-forward. One special case if DROP TABLE is running at the same time, the MDL_EXPLICIT lock may succeed *after* drop table succeed, and Rdbl_ddl_manager::find would return null, and we would give default values in the columns and with a error message saying table cannot be found.

Note this version only overrides get_table_statistics (and it updates SQL layer to treat them separately) and we can add get_index_column_cardinality in the future if there is a need - right now there is little evidence that we are querying information_schema.statistics much (if not at all).

Reviewed By: hermanlee

Differential Revision: D26624028

Author: yizhang82

mysql-test/suite/rocksdb/r/stats_during_drop_table.result

@@ -0,0 +1,38 @@
+set @old_debug = @@global.debug;
+drop table if exists t1;
+set @@global.debug = '+d,rocksdb_before_delete_table';
+create table t1 (
+id1 int(10) unsigned not null default '0',
+primary key (id1) comment 'cf_primary_key'
+  ) engine=rocksdb;
+Warnings:
+Warning	1681	Integer display width is deprecated and will be removed in a future release.
+drop table t1;
+set debug_sync = "now wait_for ready_to_mark_cf_dropped_before_delete_table";
+SHOW TABLE STATUS LIKE 't1';
+set debug_sync = "now signal mark_cf_dropped_done_before_delete_table";
+show tables;
+Tables_in_test
+Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
+t1	ROCKSDB	10	Fixed	4294967295	0	0	0	0	0	NULL	#	#	NULL	#	NULL		No such table: 'Table is missing'
+Warnings:
+Warning	155	No such table: 'Table is missing'
+set @@global.debug = @old_debug;
+set @@global.debug = '+d,rocksdb_after_delete_table';
+create table t1 (
+id1 int(10) unsigned not null default '0',
+primary key (id1) comment 'cf_primary_key'
+  ) engine=rocksdb;
+Warnings:
+Warning	1681	Integer display width is deprecated and will be removed in a future release.
+drop table t1;
+set debug_sync = "now wait_for ready_to_mark_cf_dropped_after_delete_table";
+SHOW TABLE STATUS LIKE 't1';
+set debug_sync = "now signal mark_cf_dropped_done_after_delete_table";
+show tables;
+Tables_in_test
+Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
+t1	ROCKSDB	10	Fixed	4294967295	0	0	0	0	0	NULL	#	#	NULL	#	NULL		No such table: 'Table is missing'
+Warnings:
+Warning	155	No such table: 'Table is missing'
+set @@global.debug = @old_debug;

mysql-test/suite/rocksdb/r/truncate_table.result

@@ -9,6 +9,12 @@ DROP TABLE t1;
 CREATE TABLE t1 (a INT KEY AUTO_INCREMENT, c CHAR(8)) ENGINE=RocksDB CHARSET=latin1;
 SHOW TABLE STATUS LIKE 't1';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
+t1	ROCKSDB	10	Fixed	#	#	#	0	0	0	NULL	#	#	NULL	latin1_swedish_ci	NULL		
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+0
+SHOW TABLE STATUS LIKE 't1';
+Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
 t1	ROCKSDB	10	Fixed	#	#	#	0	0	0	1	#	#	NULL	latin1_swedish_ci	NULL		
 INSERT INTO t1 (c) VALUES ('a'),('b'),('c');
 SHOW TABLE STATUS LIKE 't1';
@@ -17,6 +23,12 @@ t1	ROCKSDB	10	Fixed	#	#	#	0	0	0	4	#	#	NULL	latin1_swedish_ci	NULL
 TRUNCATE TABLE t1;
 SHOW TABLE STATUS LIKE 't1';
 Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
+t1	ROCKSDB	10	Fixed	#	#	#	0	0	0	NULL	#	#	NULL	latin1_swedish_ci	NULL		
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+0
+SHOW TABLE STATUS LIKE 't1';
+Name	Engine	Version	Row_format	Rows	Avg_row_length	Data_length	Max_data_length	Index_length	Data_free	Auto_increment	Create_time	Update_time	Check_time	Collation	Checksum	Create_options	Comment
 t1	ROCKSDB	10	Fixed	#	#	#	0	0	0	1	#	#	NULL	latin1_swedish_ci	NULL		
 INSERT INTO t1 (c) VALUES ('d');
 SHOW TABLE STATUS LIKE 't1';

mysql-test/suite/rocksdb/t/stats_during_drop_table.test

@@ -0,0 +1,88 @@
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/have_rocksdb.inc
+--source include/count_sessions.inc
+
+--disable_query_log
+#call mtr.add_suppression("Cannot mark Column family.*because it is in use");
+--enable_query_log
+
+set @old_debug = @@global.debug;
+
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+connect (conn1,localhost,root,,);
+connect (conn2,localhost,root,,);
+connect (conn3,localhost,root,,);
+
+## test1 ##
+--connection conn1
+set @@global.debug = '+d,rocksdb_before_delete_table';
+create table t1 (
+  id1 int(10) unsigned not null default '0',
+  primary key (id1) comment 'cf_primary_key'
+  ) engine=rocksdb;
+
+send drop table t1;
+
+--connection conn2
+set debug_sync = "now wait_for ready_to_mark_cf_dropped_before_delete_table";
+
+# replace_column is in reap
+send SHOW TABLE STATUS LIKE 't1';
+
+--connection conn3
+let $wait_condition= SELECT count(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE State = 'Waiting for table metadata lock';
+--source include/wait_condition.inc
+set debug_sync = "now signal mark_cf_dropped_done_before_delete_table";
+
+--connection conn1
+reap;
+show tables;
+
+--connection conn2
+--replace_column 12 # 13 # 15 #
+reap;
+
+set @@global.debug = @old_debug;
+
+## test2 ##
+--connection conn1
+
+set @@global.debug = '+d,rocksdb_after_delete_table';
+create table t1 (
+  id1 int(10) unsigned not null default '0',
+  primary key (id1) comment 'cf_primary_key'
+  ) engine=rocksdb;
+
+send drop table t1;
+
+--connection conn2
+set debug_sync = "now wait_for ready_to_mark_cf_dropped_after_delete_table";
+
+# replace_column is in reap
+send SHOW TABLE STATUS LIKE 't1';
+
+--connection conn3
+let $wait_condition= SELECT count(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE State = 'Waiting for table metadata lock';
+--source include/wait_condition.inc
+
+set debug_sync = "now signal mark_cf_dropped_done_after_delete_table";
+
+--connection conn1
+reap;
+show tables;
+
+--connection conn2
+--replace_column 12 # 13 # 15 #
+reap;
+
+set @@global.debug = @old_debug;
+
+--connection default
+--disconnect conn1
+--disconnect conn2
+--disconnect conn3
+--source include/wait_until_count_sessions.inc

mysql-test/suite/rocksdb/t/truncate_table.test

@@ -28,6 +28,14 @@ DROP TABLE t1;
 
 CREATE TABLE t1 (a INT KEY AUTO_INCREMENT, c CHAR(8)) ENGINE=RocksDB CHARSET=latin1;
 
+# NOTE: Before table open AUTO_INCREMENT will be NULL as table open updates it
+#--replace_column 2 # 3 # 4 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # 13 # 14 # 15 # 16 # 17 # 18 #
+--replace_column 5 # 6 # 7 # 12 # 13 #
+SHOW TABLE STATUS LIKE 't1';
+
+# Force open the table to update autoinc number 
+SELECT COUNT(*) FROM t1;
+
 #--replace_column 2 # 3 # 4 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # 13 # 14 # 15 # 16 # 17 # 18 #
 --replace_column 5 # 6 # 7 # 12 # 13 #
 SHOW TABLE STATUS LIKE 't1';
@@ -38,6 +46,15 @@ INSERT INTO t1 (c) VALUES ('a'),('b'),('c');
 SHOW TABLE STATUS LIKE 't1';
 
 TRUNCATE TABLE t1;
+
+# NOTE: Before table open AUTO_INCREMENT will be NULL as table open updates it
+#--replace_column 2 # 3 # 4 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # 13 # 14 # 15 # 16 # 17 # 18 #
+--replace_column 5 # 6 # 7 # 12 # 13 #
+
+SHOW TABLE STATUS LIKE 't1';
+# Force open the table to update autoinc number 
+SELECT COUNT(*) FROM t1;
+
 #--replace_column 2 # 3 # 4 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # 13 # 14 # 15 # 16 # 17 # 18 #
 --replace_column 5 # 6 # 7 # 12 # 13 #
 SHOW TABLE STATUS LIKE 't1';
@@ -71,4 +88,3 @@ HANDLER h2 READ FIRST;
 --enable_testcase
 
 DROP TABLE t1;
-

sql/dd/info_schema/table_stats.cc

@@ -491,9 +491,15 @@ ulonglong Table_statistics::read_stat(
   plugin_ref tmp_plugin = ha_resolve_by_name_raw(
       thd, lex_cstring_handle(dd::String_type(engine_name_ptr.ptr())));
   handlerton *hton = nullptr;
-  const bool hton_implements_get_statistics =
-      (tmp_plugin && (hton = plugin_data<handlerton *>(tmp_plugin)) &&
-       hton->get_index_column_cardinality && hton->get_table_statistics);
+  bool hton_implements_get_statistics = false;
+  if (tmp_plugin && (hton = plugin_data<handlerton *>(tmp_plugin))) {
+    if (stype == enum_table_stats_type::INDEX_COLUMN_CARDINALITY) {
+      hton_implements_get_statistics =
+          hton->get_index_column_cardinality != nullptr;
+    } else {
+      hton_implements_get_statistics = hton->get_table_statistics != nullptr;
+    }
+  }
 
   // Try to get statistics without opening the table.
   if (!partition_name && hton_implements_get_statistics)