Skip to content

Commit 39be853

Browse files
saumitrpfacebook-github-bot
saumitrp
authored and
facebook-github-bot
committed
debug flag to test iterator bounds with bloom filters
Summary: When bloom filter is configured on a column family, it may be hard to test the correctness of the new, tighter, iterator bounds for typical queries where bloom filters are applicable (since we use either bloom filters or iterator bounds). This debug flag can be used to test the iterator bounds (select correctness etc) for all kinds of queries even when bloom filters would otherwise have worked (and prevented iterator bounds from being set). Usage: SET global ROCKSDB_DEBUG_SKIP_BLOOM_FILTER_CHECK_ON_ITERATOR_BOUNDS = 1; Differential Revision: D56591849 fbshipit-source-id: c55b04c
1 parent f109280 commit 39be853

11 files changed

+411
-153
lines changed

‎mysql-test/suite/rocksdb/r/bloomfilter_skip2.result‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
CREATE PROCEDURE bloom_start()
2+
BEGIN
3+
select sum(variable_value) into @u from performance_schema.global_status
4+
where variable_name in (
5+
'rocksdb_bloom_filter_prefix_useful',
6+
'rocksdb_last_level_seek_filter_match',
7+
'rocksdb_last_level_seek_filtered',
8+
'rocksdb_last_level_seek_data_useful_filter_match',
9+
'rocksdb_non_last_level_seek_filter_match',
10+
'rocksdb_non_last_level_seek_filtered',
11+
'rocksdb_non_last_level_seek_data_useful_filter_match');
12+
END//
13+
CREATE PROCEDURE bloom_eq_check()
14+
BEGIN
15+
select case when sum(variable_value)-@u = 0 then 'true' else 'false' end as useful from performance_schema.global_status
16+
where variable_name in (
17+
'rocksdb_bloom_filter_prefix_useful',
18+
'rocksdb_last_level_seek_filter_match',
19+
'rocksdb_last_level_seek_filtered',
20+
'rocksdb_last_level_seek_data_useful_filter_match',
21+
'rocksdb_non_last_level_seek_filter_match',
22+
'rocksdb_non_last_level_seek_filtered',
23+
'rocksdb_non_last_level_seek_data_useful_filter_match');
24+
END//
25+
CREATE PROCEDURE bloom_gt_check()
26+
BEGIN
27+
select case when sum(variable_value)-@u > 0 then 'true' else 'false' end as useful from performance_schema.global_status
28+
where variable_name in (
29+
'rocksdb_bloom_filter_prefix_useful',
30+
'rocksdb_last_level_seek_filter_match',
31+
'rocksdb_last_level_seek_filtered',
32+
'rocksdb_last_level_seek_data_useful_filter_match',
33+
'rocksdb_non_last_level_seek_filter_match',
34+
'rocksdb_non_last_level_seek_filtered',
35+
'rocksdb_non_last_level_seek_data_useful_filter_match');
36+
END//
37+
CREATE TABLE t0 (id1 VARCHAR(30), id2 INT, value INT, PRIMARY KEY (id1, id2)) ENGINE=rocksdb collate latin1_bin;
38+
call bloom_start();
39+
SELECT COUNT(*) FROM t0 WHERE id1='X' AND id2>=1;
40+
COUNT(*)
41+
10000
42+
call bloom_eq_check();
43+
useful
44+
true
45+
DROP TABLE t0;
46+
CREATE TABLE t1 (id1 BIGINT, id2 INT, id3 BIGINT, value INT, PRIMARY KEY (id1, id2, id3)) ENGINE=rocksdb;
47+
call bloom_start();
48+
SELECT COUNT(*) FROM t1 WHERE id1=1 AND id2=1 AND id3>=2;
49+
COUNT(*)
50+
9999
51+
call bloom_eq_check();
52+
useful
53+
true
54+
call bloom_start();
55+
SELECT COUNT(*) FROM t1 WHERE id1=1 AND id2>=1 AND id3>=2;
56+
COUNT(*)
57+
9999
58+
call bloom_eq_check();
59+
useful
60+
true
61+
DROP TABLE t1;
62+
CREATE TABLE t2 (id1 INT, id2 VARCHAR(100), id3 BIGINT, value INT, PRIMARY KEY (id1, id2, id3)) ENGINE=rocksdb collate latin1_bin;
63+
call bloom_start();
64+
select count(*) from t2 WHERE id1=100 and id2 IN ('00000000000000000000', '100');
65+
count(*)
66+
1
67+
call bloom_gt_check();
68+
useful
69+
true
70+
call bloom_start();
71+
select count(*) from t2 WHERE id1=200 and id2 IN ('00000000000000000000', '200');
72+
count(*)
73+
1
74+
call bloom_gt_check();
75+
useful
76+
true
77+
call bloom_start();
78+
select count(*) from t2 WHERE id1=200 and id2 IN ('3', '200');
79+
count(*)
80+
1
81+
call bloom_eq_check();
82+
useful
83+
true
84+
DROP TABLE t2;
85+
CREATE TABLE t3 (id1 BIGINT, id2 BIGINT, id3 BIGINT, id4 BIGINT, PRIMARY KEY (id1, id2, id3, id4)) ENGINE=rocksdb collate latin1_bin;
86+
call bloom_start();
87+
SELECT COUNT(*) FROM t3 WHERE id1=1 AND id2=5000 AND id3=1 AND id4=1;
88+
COUNT(*)
89+
0
90+
call bloom_gt_check();
91+
useful
92+
true
93+
call bloom_start();
94+
SELECT COUNT(*) FROM t3 WHERE id1=1 AND id2=1 AND id3=1;
95+
COUNT(*)
96+
1
97+
call bloom_gt_check();
98+
useful
99+
true
100+
call bloom_start();
101+
SELECT COUNT(*) FROM t3 WHERE id1=1 AND id2=1 AND id3=1 AND id4 <= 500;
102+
COUNT(*)
103+
1
104+
call bloom_gt_check();
105+
useful
106+
true
107+
DROP TABLE t3;
108+
DROP PROCEDURE bloom_start;
109+
DROP PROCEDURE bloom_eq_check;
110+
DROP PROCEDURE bloom_gt_check;

‎mysql-test/suite/rocksdb/r/rocksdb.result‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -959,6 +959,7 @@ rocksdb_debug_binlog_ttl_compaction_ts_delta 0
959959
rocksdb_debug_cardinality_multiplier 2
960960
rocksdb_debug_manual_compaction_delay 0
961961
rocksdb_debug_optimizer_no_zero_cardinality ON
962+
rocksdb_debug_skip_bloom_filter_check_on_iterator_bounds OFF
962963
rocksdb_debug_ttl_ignore_pk OFF
963964
rocksdb_debug_ttl_read_filter_ts 0
964965
rocksdb_debug_ttl_rec_ts 0

‎mysql-test/suite/rocksdb/t/bloomfilter2.inc‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
--source include/have_rocksdb.inc
2+
# with rocksdb DD, these status variable values also inclue DD tables. etc
3+
# thus these status variable value change much more
4+
--source include/have_innodb_ddse.inc
5+
6+
DELIMITER //;
7+
CREATE PROCEDURE bloom_start()
8+
BEGIN
9+
select sum(variable_value) into @u from performance_schema.global_status
10+
where variable_name in (
11+
'rocksdb_bloom_filter_prefix_useful',
12+
'rocksdb_last_level_seek_filter_match',
13+
'rocksdb_last_level_seek_filtered',
14+
'rocksdb_last_level_seek_data_useful_filter_match',
15+
'rocksdb_non_last_level_seek_filter_match',
16+
'rocksdb_non_last_level_seek_filtered',
17+
'rocksdb_non_last_level_seek_data_useful_filter_match');
18+
END//
19+
CREATE PROCEDURE bloom_eq_check()
20+
BEGIN
21+
select case when sum(variable_value)-@u = 0 then 'true' else 'false' end as useful from performance_schema.global_status
22+
where variable_name in (
23+
'rocksdb_bloom_filter_prefix_useful',
24+
'rocksdb_last_level_seek_filter_match',
25+
'rocksdb_last_level_seek_filtered',
26+
'rocksdb_last_level_seek_data_useful_filter_match',
27+
'rocksdb_non_last_level_seek_filter_match',
28+
'rocksdb_non_last_level_seek_filtered',
29+
'rocksdb_non_last_level_seek_data_useful_filter_match');
30+
END//
31+
CREATE PROCEDURE bloom_gt_check()
32+
BEGIN
33+
select case when sum(variable_value)-@u > 0 then 'true' else 'false' end as useful from performance_schema.global_status
34+
where variable_name in (
35+
'rocksdb_bloom_filter_prefix_useful',
36+
'rocksdb_last_level_seek_filter_match',
37+
'rocksdb_last_level_seek_filtered',
38+
'rocksdb_last_level_seek_data_useful_filter_match',
39+
'rocksdb_non_last_level_seek_filter_match',
40+
'rocksdb_non_last_level_seek_filtered',
41+
'rocksdb_non_last_level_seek_data_useful_filter_match');
42+
END//
43+
DELIMITER ;//
44+
45+
## Test 0: Eq cond len includs VARCHAR, and real cond len < prefix bloom len < VARCHAR definition len
46+
CREATE TABLE t0 (id1 VARCHAR(30), id2 INT, value INT, PRIMARY KEY (id1, id2)) ENGINE=rocksdb collate latin1_bin;
47+
--disable_query_log
48+
let $i = 1;
49+
while ($i <= 10000) {
50+
let $insert = INSERT INTO t0 VALUES('X', $i, $i);
51+
inc $i;
52+
eval $insert;
53+
}
54+
--enable_query_log
55+
56+
# BF not used
57+
call bloom_start();
58+
SELECT COUNT(*) FROM t0 WHERE id1='X' AND id2>=1;
59+
call bloom_eq_check();
60+
61+
DROP TABLE t0;
62+
63+
64+
## Test 1: Eq cond len is shorter than prefix bloom len
65+
CREATE TABLE t1 (id1 BIGINT, id2 INT, id3 BIGINT, value INT, PRIMARY KEY (id1, id2, id3)) ENGINE=rocksdb;
66+
67+
--disable_query_log
68+
let $i = 1;
69+
while ($i <= 10000) {
70+
let $insert = INSERT INTO t1 VALUES(1, 1, $i, $i);
71+
eval $insert;
72+
inc $i;
73+
}
74+
--enable_query_log
75+
76+
# BF not used (4+8+4=16)
77+
call bloom_start();
78+
SELECT COUNT(*) FROM t1 WHERE id1=1 AND id2=1 AND id3>=2;
79+
call bloom_eq_check();
80+
81+
# BF not used (4+8=12)
82+
call bloom_start();
83+
SELECT COUNT(*) FROM t1 WHERE id1=1 AND id2>=1 AND id3>=2;
84+
call bloom_eq_check();
85+
86+
DROP TABLE t1;
87+
88+
89+
## Test 2: Long IN and short IN (varchar) -- can_use_bloom_filter changes within the same query
90+
CREATE TABLE t2 (id1 INT, id2 VARCHAR(100), id3 BIGINT, value INT, PRIMARY KEY (id1, id2, id3)) ENGINE=rocksdb collate latin1_bin;
91+
--disable_query_log
92+
let $i = 1;
93+
while ($i <= 10000) {
94+
let $insert = INSERT INTO t2 VALUES($i, $i, $i, $i);
95+
inc $i;
96+
eval $insert;
97+
}
98+
--enable_query_log
99+
100+
# BF used for large cond, not used for short cond
101+
call bloom_start();
102+
select count(*) from t2 WHERE id1=100 and id2 IN ('00000000000000000000', '100');
103+
call bloom_gt_check();
104+
105+
call bloom_start();
106+
select count(*) from t2 WHERE id1=200 and id2 IN ('00000000000000000000', '200');
107+
call bloom_gt_check();
108+
109+
# BF not used because cond length is too small in all cases
110+
call bloom_start();
111+
select count(*) from t2 WHERE id1=200 and id2 IN ('3', '200');
112+
call bloom_eq_check();
113+
114+
DROP TABLE t2;
115+
116+
117+
## Test 3: Eq cond len is longer than prefix bloom len
118+
CREATE TABLE t3 (id1 BIGINT, id2 BIGINT, id3 BIGINT, id4 BIGINT, PRIMARY KEY (id1, id2, id3, id4)) ENGINE=rocksdb collate latin1_bin;
119+
--disable_query_log
120+
let $i = 1;
121+
while ($i <= 10000) {
122+
if ($i != 5000) {
123+
let $insert = INSERT INTO t3 VALUES(1, $i, $i, $i);
124+
eval $insert;
125+
}
126+
inc $i;
127+
}
128+
--enable_query_log
129+
130+
# Full BF works with Get(), Block based does not.
131+
call bloom_start();
132+
SELECT COUNT(*) FROM t3 WHERE id1=1 AND id2=5000 AND id3=1 AND id4=1;
133+
call bloom_gt_check();
134+
135+
# BF used (4+8+8+8)
136+
call bloom_start();
137+
SELECT COUNT(*) FROM t3 WHERE id1=1 AND id2=1 AND id3=1;
138+
call bloom_gt_check();
139+
140+
call bloom_start();
141+
SELECT COUNT(*) FROM t3 WHERE id1=1 AND id2=1 AND id3=1 AND id4 <= 500;
142+
call bloom_gt_check();
143+
144+
DROP TABLE t3;
145+
146+
DROP PROCEDURE bloom_start;
147+
DROP PROCEDURE bloom_eq_check;
148+
DROP PROCEDURE bloom_gt_check;

0 commit comments

Comments
 (0)