More precise estimation of column size for ColumnString

Andded PrettyPrintByteSize + some tests

Author: Enmk

ut/ColumnString_ut.cpp

@@ -8,6 +8,7 @@
 #include "utils.h"
 #include "value_generators.h"
 
+#include <algorithm>
 #include <cstddef>
 #include <initializer_list>
 #include <ios>
@@ -38,13 +39,25 @@ size_t EstimateColumnStringMemoryUsage(
     if (item_estimated_size == ColumnString::EstimatedValueSize{0} && total_items_size && number_of_items)
         item_estimated_size = ColumnString::EstimatedValueSize(static_cast<double>(*total_items_size) / number_of_items);
 
-    return number_of_items * sizeof(std::string_view)
-        + number_of_items * static_cast<size_t>(item_estimated_size) * value_to_estimation_average_size_ratio
+    const size_t estimated_total_item_size = number_of_items * static_cast<size_t>(item_estimated_size) * value_to_estimation_average_size_ratio;
+    const auto estimated_number_of_blocks = std::max<size_t>(1, estimated_total_item_size ? COLUMN_STRING_DEFAULT_BLOCK_SIZE / estimated_total_item_size : 1);
+
+    // space wasted in block since not all items can be fit perfectly, and there is some unused space at the end of the block.
+    const auto estimate_lost_space_in_block = (static_cast<size_t>(item_estimated_size) != 0
+            ? COLUMN_STRING_DEFAULT_BLOCK_SIZE % static_cast<size_t>(static_cast<size_t>(item_estimated_size) * value_to_estimation_average_size_ratio)
+            : COLUMN_STRING_DEFAULT_BLOCK_SIZE / 10);
+
+    const auto max_estimation_error_factor = item_estimated_size == ColumnString::NO_PREALLOCATE ? 2.5 : 2;
+
+    return (number_of_items * sizeof(std::string_view)
+        + estimated_total_item_size
+        + estimate_lost_space_in_block * estimated_number_of_blocks
         + COLUMN_STRING_DEFAULT_BLOCK_SIZE
         // It is hard to compute overhead added by vector<ColumnString::Block>
         // (mostly because we don't know number of ColumnString::Block instances from outside, and this number depends on many factors),
         // so we just make a guess.
-        + COLUMN_STRING_MAX_EXPECTED_MEMORY_OVERHEAD;
+            + COLUMN_STRING_MAX_EXPECTED_MEMORY_OVERHEAD)
+        * max_estimation_error_factor;
 }
 
 std::string ScaleString(std::string str, size_t required_size) {
@@ -289,8 +302,15 @@ struct ColumnStringEstimatedValueSizeTest : public ::testing::TestWithParam<std:
         // Adjust number of items so the test doesn't use too much memory
         if (static_cast<size_t>(single_value_size_estimation) != 0
             // *2 since we store both reference values and values in column itself.
-                && EstimateColumnStringMemoryUsage(expected_number_of_items, single_value_size_estimation, size_ratio.average) * 2 > MAX_MEMORY_USAGE) {
+                && EstimateColumnStringMemoryUsage(expected_number_of_items, single_value_size_estimation, size_ratio.average) > MAX_MEMORY_USAGE) {
+            const auto old_expected_number_of_items = expected_number_of_items;
             expected_number_of_items = MAX_MEMORY_USAGE / (static_cast<size_t>(single_value_size_estimation) * 2 * size_ratio.average);
+
+            std::cerr << "To avoid using too much memory, reduced number of items in test"
+                      << " from " << old_expected_number_of_items
+                      << " to " << expected_number_of_items
+                      << ", expected item size is " << single_value_size_estimation
+                      << std::endl;
         }
     }
 
@@ -318,9 +338,9 @@ struct ColumnStringEstimatedValueSizeTest : public ::testing::TestWithParam<std:
         ASSERT_TRUE(CompareRecursive(values, column));
     }
 
-    size_t EstimateMemoryUsage(size_t total_values_size) {
+    size_t EstimateMemoryUsage(size_t total_values_size, float expected_number_of_items_multiplier = 1.0) {
         const auto & [single_value_size_estimation, size_ratio] = GetParam();
-        return EstimateColumnStringMemoryUsage(expected_number_of_items, single_value_size_estimation, size_ratio.average, total_values_size);
+        return EstimateColumnStringMemoryUsage(expected_number_of_items * expected_number_of_items_multiplier, single_value_size_estimation, size_ratio.average, total_values_size);
     }
 };
 
@@ -363,8 +383,10 @@ TEST_P(ColumnStringEstimatedValueSizeTest, AppendNoReserve)
 
     EXPECT_NO_FATAL_FAILURE(AppendStrings(col, total_values_size));
 
-    // since there was no Reserve call prior, there could be more some overallocations, hence *2
-    EXPECT_LT(col.MemoryUsage(), EstimateMemoryUsage(total_values_size) * 2);
+    const auto max_estimation_error_factor = single_value_size_estimation == ColumnString::NO_PREALLOCATE ? 2.5 : 2;
+
+    // since there was no Reserve call prior, there could be more some overallocations, hence some estimation error
+    EXPECT_LT(col.MemoryUsage(), EstimateMemoryUsage(total_values_size) * max_estimation_error_factor);
 }
 
 TEST_P(ColumnStringEstimatedValueSizeTest, ReserveExactAndAppend)
@@ -377,8 +399,10 @@ TEST_P(ColumnStringEstimatedValueSizeTest, ReserveExactAndAppend)
     EXPECT_NO_THROW(col.Reserve(expected_number_of_items));
     EXPECT_NO_FATAL_FAILURE(AppendStrings(col, total_values_size));
 
+    const auto max_estimation_error_factor = single_value_size_estimation == ColumnString::NO_PREALLOCATE ? 2.5 : 2;
+
     // Allow minor overallocations, hence * 1.2
-    EXPECT_LT(col.MemoryUsage(), EstimateMemoryUsage(total_values_size) * 2);
+    EXPECT_LT(col.MemoryUsage(), EstimateMemoryUsage(total_values_size) * max_estimation_error_factor);
 }
 
 TEST_P(ColumnStringEstimatedValueSizeTest, ReserveLessAndAppend)
@@ -391,8 +415,10 @@ TEST_P(ColumnStringEstimatedValueSizeTest, ReserveLessAndAppend)
     EXPECT_NO_THROW(col.Reserve(expected_number_of_items * .8));
     EXPECT_NO_FATAL_FAILURE(AppendStrings(col, total_values_size));
 
+    const auto max_estimation_error_factor = single_value_size_estimation == ColumnString::NO_PREALLOCATE ? 2.5 : 2;
+
     // Allow minor overallocations, hence * 1.2
-    EXPECT_LT(col.MemoryUsage(), EstimateMemoryUsage(total_values_size) * 2);
+    EXPECT_LT(col.MemoryUsage(), EstimateMemoryUsage(total_values_size) * max_estimation_error_factor);
 }
 
 TEST_P(ColumnStringEstimatedValueSizeTest, ReserveMoreAndAppend)
@@ -405,8 +431,10 @@ TEST_P(ColumnStringEstimatedValueSizeTest, ReserveMoreAndAppend)
     EXPECT_NO_THROW(col.Reserve(expected_number_of_items * 1.2));
     EXPECT_NO_FATAL_FAILURE(AppendStrings(col, total_values_size));
 
+    const auto max_estimation_error_factor = single_value_size_estimation == ColumnString::NO_PREALLOCATE ? 2.5 : 2;
+
     // Allow minor overallocations, hence * 1.2
-    EXPECT_LT(col.MemoryUsage(), EstimateColumnStringMemoryUsage(expected_number_of_items * 1.2, single_value_size_estimation, size_ratio.average, total_values_size) * 2);
+    EXPECT_LT(col.MemoryUsage(), EstimateMemoryUsage(total_values_size, 1.2) * max_estimation_error_factor);
 }
 
 /** TODO more tests

ut/utils.cpp

@@ -20,7 +20,9 @@
 
 #include <cinttypes>
 #include <iomanip>
+#include <ios>
 #include <sstream>
+#include <string_view>
 #include <type_traits>
 
 
@@ -256,6 +258,48 @@ std::ostream& operator<<(std::ostream & ostr, const PrettyPrintBlock & pretty_pr
     return ostr;
 }
 
+std::ostream& operator<<(std::ostream & ostr, const PrettyPrintByteSize & byte_size) {
+    static const std::pair<size_t, const char *> FACTORS[] = {
+        { 1,              "bytes" },
+        { 1024,           "KiB"   },
+        { 1024*1024,      "MiB"   },
+        { 1024*1024*1024, "GiB"   },
+    };
+
+    auto p = std::find_if(std::begin(FACTORS), std::end(FACTORS), [&byte_size](const auto v) {
+        return byte_size.bytes < v.first;
+    });
+
+    if (p != std::begin(FACTORS)) {
+        --p;
+    }
+
+    const float resulting_size = byte_size.bytes / static_cast<float>(p->first);
+
+    // Trim trailing zeroes after decimal point, if present.
+    {
+        std::stringstream sstr;
+        sstr << std::fixed << std::setprecision(byte_size.max_decimal_points) << resulting_size;
+
+        auto s = sstr.str();
+
+        // here we completely ignore locales and just assume that '.' is used as decimal point
+        const auto decimal_point_position = s.find_last_of('.');
+        const auto last_non_zero_decimal_number_pos = s.find_last_not_of('0');
+
+        if (decimal_point_position != s.npos && last_non_zero_decimal_number_pos != s.npos) {
+            if (decimal_point_position == last_non_zero_decimal_number_pos)
+                s.erase(decimal_point_position);
+            else
+                s.erase(std::max(decimal_point_position, last_non_zero_decimal_number_pos + 1));
+        }
+
+        ostr << s;
+    }
+
+    return ostr << " " << p->second;
+}
+
 std::ostream& operator<<(std::ostream& ostr, const in_addr& addr) {
     char buf[INET_ADDRSTRLEN];
     const char* ip_str = inet_ntop(AF_INET, &addr, buf, sizeof(buf));
@@ -332,6 +376,28 @@ std::ostream & operator<<(std::ostream & ostr, const Progress & progress) {
         << " written_bytes : " << progress.written_bytes;
 }
 
+std::ostream & operator<<(std::ostream & ostr, const ColumnString::EstimatedValueSize & estimation) {
+    static const std::pair<ColumnString::EstimatedValueSize, const char *> NAMES_OF_DEFAULT_VALUES[] = {
+        { ColumnString::NO_PREALLOCATE,              "DO NOT PREALLOCATE" },
+        { ColumnString::EstimatedValueSize::TINY,    "TINY (8 bytes)"     },
+        { ColumnString::EstimatedValueSize::SMALL,   "SMALL (32 bytes)"   },
+        { ColumnString::EstimatedValueSize::MEDIUM,  "MEDIUM (128 bytes)" },
+        { ColumnString::EstimatedValueSize::LARGE,   "LARGE (512 bytes)"  }
+    };
+
+    const auto p = std::find_if(std::begin(NAMES_OF_DEFAULT_VALUES), std::end(NAMES_OF_DEFAULT_VALUES), [&estimation](const auto v) {
+        return v.first == estimation;
+    });
+
+    ostr << "ColumnString::EstimatedValueSize{ ";
+    if (p != std::end(NAMES_OF_DEFAULT_VALUES))
+        ostr << p->second;
+    else
+        ostr << PrettyPrintByteSize{static_cast<size_t>(estimation)};
+
+    return ostr << " }";
+}
+
 }
 
 uint64_t versionNumber(const ServerInfo & server_info) {
@@ -350,5 +416,5 @@ std::string ToString(const clickhouse::UUID& v) {
 }
 
 std::ostream & dumpMemoryUsage(const char * prefix, const clickhouse::ColumnRef col) {
-    return std::cerr << prefix << " " << col->GetType().GetName() << " : " << col->MemoryUsage() << " bytes" << std::endl;
+    return std::cerr << prefix << " " << col->GetType().GetName() << " : " << PrettyPrintByteSize{col->MemoryUsage()} << std::endl;
 }

ut/utils.h

@@ -141,19 +141,25 @@ struct PrettyPrintBlock {
     const clickhouse::Block & block;
 };
 
+// Print byte size in either in bytes, KiB, MiB, or GiB.
+struct PrettyPrintByteSize {
+    size_t bytes;
+    size_t max_decimal_points = 3;
+};
+
 namespace clickhouse {
 std::ostream& operator<<(std::ostream & ostr, const Block & block);
 std::ostream& operator<<(std::ostream & ostr, const Type & type);
 std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info);
 std::ostream & operator<<(std::ostream & ostr, const Profile & profile);
 std::ostream & operator<<(std::ostream & ostr, const Progress & progress);
-inline std::ostream & operator<<(std::ostream & ostr, const ColumnString::EstimatedValueSize & estimation) {
-    return ostr << "ColumnString::EstimatedValueSize{ " << static_cast<std::underlying_type_t<ColumnString::EstimatedValueSize>>(estimation) << " }";
-}
+std::ostream & operator<<(std::ostream & ostr, const ColumnString::EstimatedValueSize & estimation);
 
 }
 
 std::ostream& operator<<(std::ostream & ostr, const PrettyPrintBlock & block);
+std::ostream& operator<<(std::ostream & ostr, const PrettyPrintByteSize & block);
+
 std::ostream& operator<<(std::ostream& ostr, const in_addr& addr);
 std::ostream& operator<<(std::ostream& ostr, const in6_addr& addr);
 

ut/utils_ut.cpp

@@ -2,6 +2,7 @@
 #include "ut/value_generators.h"
 #include "utils.h"
 
+#include <initializer_list>
 #include <limits>
 #include <optional>
 #include <vector>
@@ -120,3 +121,56 @@ TEST(Generators, MakeArrays) {
     auto arrays = MakeArrays<std::string, MakeStrings>();
     ASSERT_LT(0u, arrays.size());
 }
+
+class OutputTest : public ::testing::Test {
+public:
+    template <typename T>
+    static std::string ToString(const T & t) {
+        std::stringstream sstr;
+        sstr << t;
+
+        return sstr.str();
+    }
+};
+
+TEST_F(OutputTest, PrettyPrintByteSize)
+{
+    EXPECT_EQ("3 bytes", ToString(PrettyPrintByteSize{3}));
+
+    EXPECT_EQ("30 bytes", ToString(PrettyPrintByteSize{30}));
+    EXPECT_EQ("300 bytes", ToString(PrettyPrintByteSize{300}));
+
+    EXPECT_EQ("123 bytes", ToString(PrettyPrintByteSize{123}));
+
+    for (const auto & [base, base_name] : std::initializer_list<std::pair<size_t, const char*>>{
+            // {1,               "bytes"},
+            {1024,            "KiB"},
+            {1024*1024,       "MiB"},
+            {1024*1024*1024,  "GiB"},
+         } )
+    {
+        for (const auto & [value, value_str] : std::initializer_list<std::pair<float, const char*>>{
+                {1, "1"},
+                {1.01, "1.01"},
+                {1.10, "1.1"},
+                {1.5, "1.5"},
+                {3, "3"},
+                {3.25, "3.25"},
+                {13.75, "13.75"},
+                {135.5, "135.5"},
+                {135.125, "135.125"},
+                {10, "10"},
+                {100, "100"},
+                {1000, "1000"},
+             })
+        {
+            const auto bytes_value = static_cast<size_t>(base * value);
+            const auto expected_str = std::string(value_str) + " " + base_name;
+            EXPECT_EQ(expected_str, ToString(PrettyPrintByteSize{bytes_value}))
+                << "\n\tbase:      " << base
+                << "\n\tbase_name: " << base_name
+                << "\n\tvalue:     " << value
+                << "\n\tvalue_str: " << value_str;
+        }
+    }
+}