Cleanup rdb_hexdump (#1442)

Summary:
Currently, `rdb_hexdump` has a third argument `maxsize` with a default value of 0,
meaning unlimited output length, and there is also a `RDB_MAX_HEXDUMP` constant
for the callers to use. However, only a part of the callers use it, the rest use
the default or something else altogether (i.e. `FN_REFLEN`)

Change as follows:
- Default to `RDB_MAX_HEXDUMP` `maxsize` instead of unlimited. This should cover
  all the reasonable cases, and prevent runaway dumping in the case of corrupted
  `data_len` values.
- Bump `RDB_MAX_HEXDUMP_LEN` from 1000 to 7000. This way it can dump any MyRocks
  key (3072 byte maximum). Make it `constexpr`.

Minor cleanups:
- Where `std::string` is used as an input for `rdb_hexdump`, pass its buffer
  through `data()`, not through `c_str()`, because it's used as a sized buffer
  and not as a zero-terminated C string
- Mark `rdb_hexdump` as `nodiscard`, make its constants `constexpr`, remove
  redundant `const` from by-value parameters.

Pull Request resolved: #1442

Differential Revision: D55594025

fbshipit-source-id: d048078

Author: laurynas-biveinis

mysql-test/suite/rocksdb/r/sys_tables_acl_tables_row_locking.result

@@ -6645,7 +6645,7 @@ class Rdb_snapshot_status : public Rdb_tx_list_walker {
                            : "NOT FOUND; CF_ID: " + std::to_string(txn.m_cf_id);
 
     txn_data.waiting_key =
-        rdb_hexdump(txn.m_waiting_key.c_str(), txn.m_waiting_key.length());
+        rdb_hexdump(txn.m_waiting_key.data(), txn.m_waiting_key.length(), 0);
 
     txn_data.exclusive_lock = txn.m_exclusive;
 
@@ -11484,20 +11484,17 @@ int ha_rocksdb::check(THD *const thd MY_ATTRIBUTE((__unused__)),
 
       print_and_error : {
         std::string buf;
-        buf = rdb_hexdump(rowkey_copy.ptr(), rowkey_copy.length(),
-                          RDB_MAX_HEXDUMP_LEN);
+        buf = rdb_hexdump(rowkey_copy.ptr(), rowkey_copy.length());
         // NO_LINT_DEBUG
         LogPluginErrMsg(ERROR_LEVEL, ER_LOG_PRINTF_MSG,
                         "CHECKTABLE %s:   rowkey: %s", table_name, buf.c_str());
 
-        buf = rdb_hexdump(m_retrieved_record.data(), m_retrieved_record.size(),
-                          RDB_MAX_HEXDUMP_LEN);
+        buf = rdb_hexdump(m_retrieved_record.data(), m_retrieved_record.size());
         // NO_LINT_DEBUG
         LogPluginErrMsg(ERROR_LEVEL, ER_LOG_PRINTF_MSG,
                         "CHECKTABLE %s:   record: %s", table_name, buf.c_str());
 
-        buf = rdb_hexdump(sec_key_copy.ptr(), sec_key_copy.length(),
-                          RDB_MAX_HEXDUMP_LEN);
+        buf = rdb_hexdump(sec_key_copy.ptr(), sec_key_copy.length());
         // NO_LINT_DEBUG
         LogPluginErrMsg(ERROR_LEVEL, ER_LOG_PRINTF_MSG,
                         "CHECKTABLE %s:   index: %s", table_name, buf.c_str());

storage/rocksdb/ha_rocksdb.cc

@@ -189,8 +189,7 @@ class Rdb_compact_filter : public rocksdb::CompactionFilter {
     Rdb_string_reader reader(&existing_value);
     if (!reader.read(m_ttl_offset) || reader.read_uint64(&ttl_timestamp)) {
       std::string buf;
-      buf = rdb_hexdump(existing_value.data(), existing_value.size(),
-                        RDB_MAX_HEXDUMP_LEN);
+      buf = rdb_hexdump(existing_value.data(), existing_value.size());
       rdb_fatal_error(
           "Decoding ttl from PK value failed in compaction filter, "
           "for index (%u,%u), val: %s",

storage/rocksdb/rdb_compact_filter.h

@@ -1764,7 +1764,7 @@ void Rdb_key_def::report_checksum_mismatch(const bool is_key,
                   "Checksum mismatch in %s of key-value pair for index 0x%x",
                   is_key ? "key" : "value", get_index_number());
 
-  const std::string buf = rdb_hexdump(data, data_size, RDB_MAX_HEXDUMP_LEN);
+  const auto buf = rdb_hexdump(data, data_size);
   // NO_LINT_DEBUG
   LogPluginErrMsg(ERROR_LEVEL, ER_LOG_PRINTF_MSG,
                   "Data with incorrect checksum (%" PRIu64 " bytes): %s",

storage/rocksdb/rdb_datadic.cc

@@ -2191,8 +2191,8 @@ static int rdb_i_s_lock_info_fill_table(
   for (const auto &lock : lock_info) {
     const uint32_t cf_id = lock.first;
     const auto &key_lock_info = lock.second;
-    const auto key_hexstr = rdb_hexdump(key_lock_info.key.c_str(),
-                                        key_lock_info.key.length(), FN_REFLEN);
+    const auto key_hexstr =
+        rdb_hexdump(key_lock_info.key.data(), key_lock_info.key.length());
 
     for (const auto &id : key_lock_info.ids) {
       tables->table->field[RDB_LOCKS_FIELD::COLUMN_FAMILY_ID]->store(cf_id,
@@ -2300,10 +2300,10 @@ static int rdb_i_s_trx_info_fill_table(
   const std::vector<Rdb_trx_info> &all_trx_info = rdb_get_all_trx_info();
 
   for (const auto &info : all_trx_info) {
-    auto name_hexstr =
-        rdb_hexdump(info.name.c_str(), info.name.length(), NAME_LEN);
-    auto key_hexstr = rdb_hexdump(info.waiting_key.c_str(),
-                                  info.waiting_key.length(), FN_REFLEN);
+    const auto name_hexstr =
+        rdb_hexdump(info.name.data(), info.name.length(), NAME_LEN);
+    const auto key_hexstr =
+        rdb_hexdump(info.waiting_key.data(), info.waiting_key.length());
 
     tables->table->field[RDB_TRX_FIELD::TRANSACTION_ID]->store(info.trx_id,
                                                                true);

storage/rocksdb/rdb_i_s.cc

@@ -220,17 +220,18 @@ const std::vector<std::string> parse_into_tokens(const std::string &s,
   return tokens;
 }
 
-static const std::size_t rdb_hex_bytes_per_char = 2;
-static const std::array<char, 16> rdb_hexdigit = {{'0', '1', '2', '3', '4', '5',
-                                                   '6', '7', '8', '9', 'a', 'b',
-                                                   'c', 'd', 'e', 'f'}};
+constexpr std::size_t rdb_hex_bytes_per_char = 2;
+constexpr std::array<char, 16> rdb_hexdigit = {{'0', '1', '2', '3', '4', '5',
+                                                '6', '7', '8', '9', 'a', 'b',
+                                                'c', 'd', 'e', 'f'}};
 
 /*
-  Convert data into a hex string with optional maximum length.
-  If the data is larger than the maximum length trancate it and append "..".
+  Convert data into a hex string. If the data is larger than the maximum length
+  trancate it and append "..". If maxsize is zero, the output length is
+  unlimited.
 */
-std::string rdb_hexdump(const char *data, const std::size_t data_len,
-                        const std::size_t maxsize) {
+std::string rdb_hexdump(const char *data, std::size_t data_len,
+                        std::size_t maxsize) {
   // Count the elements in the string
   std::size_t elems = data_len;
   // Calculate the amount of output needed

storage/rocksdb/rdb_utils.cc

@@ -149,10 +149,9 @@ namespace myrocks {
   rdb_check_mutex_call_result(__PRETTY_FUNCTION__, false, \
                               mysql_mutex_unlock(&m))
 
-/*
-  Generic constant.
-*/
-const size_t RDB_MAX_HEXDUMP_LEN = 1000;
+// The default limit for rdb_hexdump output length, happens to fit the longest
+// possible keys (16K)
+constexpr size_t RDB_MAX_HEXDUMP_LEN = 2 * 16 * 1024;
 
 /*
   Helper function to get an NULL terminated uchar* out of a given MySQL String.
@@ -340,9 +339,9 @@ const std::vector<std::string> parse_into_tokens(const std::string &s,
   Helper functions to populate strings.
 */
 
-std::string rdb_hexdump(const char *data, const std::size_t data_len,
-                        const std::size_t maxsize = 0)
-    MY_ATTRIBUTE((__nonnull__));
+[[nodiscard]] std::string rdb_hexdump(
+    const char *data, std::size_t data_len,
+    std::size_t maxsize = RDB_MAX_HEXDUMP_LEN);
 
 /*
   Helper function to return dir + '/' + file