Fix range mtr tests

Summary:
During the rebase to 8.0.28, the MEM_ROOT for quick range allocation has changed from using its own MEM_ROOT to using the main MEM_ROOT, which makes accounting trickier.

While normal builds will allocate memory in large chunks and satisfy requests for memory from those chunks, asan/valgrind will convert all requests for memory into explicit allocations. This means that we allocate memory more frequently in asan/valgrind which means that we perform more frequent checks to see if we've exceeded the limit.

This leads to 2 issues:

- The main MEM_ROOT will already have memory allocated on it, which may already exceed the limit. The next allocation will always fail in that case. The fix is to set the limit to the already allocated amount plus the range max mem limit. There's some additional code in the error handler to ensure that the correct error message is displayed.
- It's generally non-deterministic whether a query will fail or not. This is because there is still unused allocated memory on the main MEM_ROOT, and depending on how much we have left and how much the range optimizer takes, we may or may not hit an error. To fix this, I just disabled the `main.range_with_memory_limit_error` test in asan/valgrind.

Squash with: D34256842

Reviewed By: hermanlee

Differential Revision: D42949953

fbshipit-source-id: bfc64bf

Author: Manuel Ung

mysql-test/t/range_with_memory_limit_error.test

@@ -2,6 +2,10 @@
 # will not be expecting errors. The error code will still be recorded in the
 # results file though.
 
+# MEM_ROOT behaves differently under valgrind/asan, so more queries will hit max mem error leading to results mismatch.
+--source include/not_valgrind.inc
+--source include/not_asan.inc
+
 --disable_abort_on_error
 set range_optimizer_max_mem_size = 10;
 set range_optimizer_fail_mode = ERROR;

sql/range_optimizer/internal.h

@@ -76,11 +76,14 @@ enum enum_range_optimizer_mem_mode { RANGE_OPT_MEM_WARN, RANGE_OPT_MEM_ERROR };
 class Range_optimizer_error_handler : public Internal_error_handler {
  public:
   Range_optimizer_error_handler()
-      : m_has_errors(false), m_is_mem_error(false) {}
+      : m_has_errors(false), m_is_mem_error(false), m_passthrough(false) {}
 
   bool handle_condition(THD *thd, uint sql_errno, const char *,
                         Sql_condition::enum_severity_level *level,
                         const char *) override {
+    // Do not handle any conditions if passthrough is set.
+    if (m_passthrough) return false;
+
     if (*level == Sql_condition::SL_ERROR) {
       m_has_errors = true;
       if (thd->variables.range_optimizer_fail_mode == RANGE_OPT_MEM_WARN) {
@@ -98,6 +101,18 @@ class Range_optimizer_error_handler : public Internal_error_handler {
               ER_THD(thd, ER_CAPACITY_EXCEEDED_IN_RANGE_OPTIMIZER));
           return true;
         }
+      } else {
+        // Call my_error with the correct range_optimizer_max_mem_size value.
+        // The default error message will contain the MEMROOT limit, which may
+        // be different from range_optimizer_max_mem_size in certain cases. To
+        // avoid infinite recursion (since we are calling my_error in the error
+        // handler), set m_passthrough to true.
+        m_passthrough = true;
+        my_error(EE_CAPACITY_EXCEEDED, MYF(0),
+                 static_cast<ulonglong>(
+                     thd->variables.range_optimizer_max_mem_size));
+        m_passthrough = false;
+        return true;
       }
     }
     return false;
@@ -108,6 +123,7 @@ class Range_optimizer_error_handler : public Internal_error_handler {
  private:
   bool m_has_errors;
   bool m_is_mem_error;
+  bool m_passthrough;
 };
 
 int index_next_different(bool is_index_scan, handler *file,

sql/range_optimizer/range_optimizer.cc

@@ -591,12 +591,14 @@ int test_quick_select(THD *thd, MEM_ROOT *return_mem_root,
 
   /* set up parameter that is passed to all functions */
   RANGE_OPT_PARAM param;
+
+  thd->push_internal_handler(&param.error_handler);
+  auto cleanup = create_scope_guard([thd] { thd->pop_internal_handler(); });
+
   if (setup_range_optimizer_param(thd, return_mem_root, temp_mem_root,
                                   keys_to_use, table, query_block, &param)) {
     return 0;
   }
-  thd->push_internal_handler(&param.error_handler);
-  auto cleanup = create_scope_guard([thd] { thd->pop_internal_handler(); });
 
   /*
     Set index_merge_allowed from OPTIMIZER_SWITCH_INDEX_MERGE.

sql/sql_optimizer.cc

@@ -6103,7 +6103,11 @@ static ha_rows get_quick_record_count(THD *thd, JOIN_TAB *tab, ha_rows limit) {
     size_t max_capacity = thd->mem_root->get_max_capacity();
     bool error_for_capacity_exceeded =
         thd->mem_root->get_error_for_capacity_exceeded();
+    // The main mem_root may already have allocated memory on it which may
+    // exceed range_optimizer_max_mem_size. Thus, set the max capacity to be the
+    // sum of the two.
     thd->mem_root->set_max_capacity(
+        thd->mem_root->allocated_size() +
         thd->variables.range_optimizer_max_mem_size);
     thd->mem_root->set_error_for_capacity_exceeded(true);