From 4e875fe6455654a7ea4549cf1eb7534b15fc59cf Mon Sep 17 00:00:00 2001 From: Benedek Kupper Date: Wed, 12 Nov 2025 21:39:28 +0100 Subject: [PATCH 1/2] backport max_serialized_size feature from develop-v4 branch --- generator/EmbeddedProto/Field.py | 13 ++- .../EmbeddedProto/templates/TypeDefMsg.h | 34 ++++++++ generator/EmbeddedProto/templates/TypeOneof.h | 23 ++++++ src/Defines.h | 9 ++- src/FieldStringBytes.h | 27 ++++++- src/Fields.h | 80 +++++++++++-------- src/ReadBufferFixedSize.h | 2 +- src/RepeatedField.h | 8 +- src/RepeatedFieldFixedSize.h | 26 ++++++ src/WireFormatter.h | 18 +++++ 10 files changed, 197 insertions(+), 43 deletions(-) diff --git a/generator/EmbeddedProto/Field.py b/generator/EmbeddedProto/Field.py index 6aa96f3..9b2d11f 100644 --- a/generator/EmbeddedProto/Field.py +++ b/generator/EmbeddedProto/Field.py @@ -157,7 +157,7 @@ def render(self, filename, jinja_environment): class FieldBasic(Field): # A dictionary to convert the wire type into a default value. type_to_default_value = {FieldDescriptorProto.TYPE_DOUBLE: "0.0", - FieldDescriptorProto.TYPE_FLOAT: "0.0", + FieldDescriptorProto.TYPE_FLOAT: "0.0f", FieldDescriptorProto.TYPE_INT64: "0", FieldDescriptorProto.TYPE_UINT64: "0U", FieldDescriptorProto.TYPE_INT32: "0", @@ -379,11 +379,18 @@ def get_type_as_defined(self): return type_name + def get_max_enum_value(self): + # Return the maximum value used by any of the enum items as a string. This is used to calculate the maximum serialized size. + max_number = self.definition.descriptor.value[0].number + for value in self.definition.descriptor.value[1:]: + if max_number < value.number: + max_number = value.number + return str(max_number) def get_type(self): - return "EmbeddedProto::enumeration<" + self.get_type_as_defined() + ">" + return "EmbeddedProto::enumeration<" + self.get_type_as_defined() + ", EmbeddedProto::WireFormatter::VarintSize(" + self.get_max_enum_value() + ")>" def get_short_type(self): - return "EmbeddedProto::enumeration<" + self.get_type_as_defined().split("::")[-1] + ">" + return "EmbeddedProto::enumeration<" + self.get_type_as_defined().split("::")[-1] + ", EmbeddedProto::WireFormatter::VarintSize(" + self.get_max_enum_value() + ")>" def get_default_value(self): return "static_cast<" + self.get_type_as_defined() + ">(0)" diff --git a/generator/EmbeddedProto/templates/TypeDefMsg.h b/generator/EmbeddedProto/templates/TypeDefMsg.h index 52a6d6c..b37dc7b 100644 --- a/generator/EmbeddedProto/templates/TypeDefMsg.h +++ b/generator/EmbeddedProto/templates/TypeDefMsg.h @@ -274,6 +274,39 @@ class {{ typedef.get_name() }} final: public ::EmbeddedProto::MessageInterface #endif + //! Calculate the maximum number of bytes required to serialize this type of message. + /*! + This function does not take into account a possible id of this message object is self. + \return The number of bytes required at most to serialize this message type. + */ + static constexpr uint32_t max_serialized_size() + { + using namespace ::EmbeddedProto; + return + {% for field in typedef.fields %} + {{"+ " if not loop.first }}{{field.get_type()}}::max_serialized_size(static_cast(FieldNumber::{{field.variable_id_name}})) + {% endfor %} + {% for oneof in typedef.oneofs %} + {{"+ " if not loop.first or typedef.fields|length != 0}}max_serialized_size_{{oneof.get_name()}}() + {% endfor %} + {% if typedef.fields|length == 0 and typedef.oneofs|length == 0 %} + 0 + {% endif %} + ; + } + + //! Calculate the maximum number of bytes required to serialize this message including the field id number and tag. + /*! + \return The number of bytes required at most to serialize this message type. + */ + static constexpr uint32_t max_serialized_size(const uint32_t field_number) + { + using namespace ::EmbeddedProto; + return WireFormatter::VarintSize(WireFormatter::MakeTag(field_number, WireFormatter::WireType::LENGTH_DELIMITED)) // Size of the tag + + WireFormatter::VarintSize(max_serialized_size()) // Length delimted value + + max_serialized_size(); // Length of the serialized content + } + #ifdef MSG_TO_STRING ::EmbeddedProto::string_view to_string(::EmbeddedProto::string_view& str) const @@ -421,6 +454,7 @@ class {{ typedef.get_name() }} final: public ::EmbeddedProto::MessageInterface {{ TypeOneof.init(oneof)|indent(6) }} {{ TypeOneof.clear(oneof)|indent(6) }} {{ TypeOneof.deserialize(oneof, environment)|indent(6) }} + {{ TypeOneof.max_serialized_size(oneof)|indent(6) }} #ifdef MSG_TO_STRING {{ TypeOneof.to_string(oneof)|indent(6) }} #endif // End of MSG_TO_STRING diff --git a/generator/EmbeddedProto/templates/TypeOneof.h b/generator/EmbeddedProto/templates/TypeOneof.h index 34ef094..899c7bf 100644 --- a/generator/EmbeddedProto/templates/TypeOneof.h +++ b/generator/EmbeddedProto/templates/TypeOneof.h @@ -157,4 +157,27 @@ ::EmbeddedProto::string_view to_string_{{_oneof.get_name()}}(::EmbeddedProto::st return left_chars; } +{% endmacro %} +{# #} +{# ------------------------------------------------------------------------------------------------------------------ #} +{# #} +{% macro max_serialized_size(_oneof) %} +static constexpr uint32_t max_serialized_size_{{_oneof.get_name()}}() +{ + using namespace EmbeddedProto; + return + {% for field in _oneof.get_fields() %} + {% if not loop.last %} + {{ ' ' * (loop.index0 * 2) }}max({{field.get_type()}}::max_serialized_size(static_cast(FieldNumber::{{field.variable_id_name}})), + {% else %} + {{ ' ' * (2 + (loop.index0 * 2))}}{{field.get_type()}}::max_serialized_size(static_cast(FieldNumber::{{field.variable_id_name}})) + {% endif %} + {% endfor %} + {% if _oneof.get_fields() | length > 1 %} + {% for i in range(_oneof.get_fields() | length - 1) %} + {{ ' ' * (5 + (loop.revindex * 2)) }}) + {% endfor %} + {% endif %} + ; +} {% endmacro %} \ No newline at end of file diff --git a/src/Defines.h b/src/Defines.h index 6c86ed0..ec03d3a 100644 --- a/src/Defines.h +++ b/src/Defines.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 Embedded AMS B.V. - All Rights Reserved + * Copyright (C) 2020-2025 Embedded AMS B.V. - All Rights Reserved * * This file is part of Embedded Proto. * @@ -16,7 +16,7 @@ * along with Embedded Proto. If not, see . * * For commercial and closed source application please visit: - * . + * . * * Embedded AMS B.V. * Info: @@ -83,6 +83,11 @@ namespace EmbeddedProto using string_view = array_view; using bytes_view = array_view; + //! Simple max function as constexpr + constexpr uint32_t max(const uint32_t a, const uint32_t b) + { + return (a > b) ? a : b; + } } #endif //_EMBEDDED_PROTO_DEFINES_H_ \ No newline at end of file diff --git a/src/FieldStringBytes.h b/src/FieldStringBytes.h index 605da20..f97ca9c 100644 --- a/src/FieldStringBytes.h +++ b/src/FieldStringBytes.h @@ -263,7 +263,32 @@ namespace EmbeddedProto void clear() override { data_.fill(0); - current_length_ = 0; + current_length_ = 0; + } + + //! When serialized with the all elements set, how much bytes are then required. + /*! + This function takes into account the field number and tag combination. + \param[in] field_number We need to include the field number. This because large field numbers require more bytes. + \return The number of bytes required at most. + */ + static constexpr uint32_t max_serialized_size(const uint32_t field_number) + { + return MAX_LENGTH // The number of bytes of the data. + + WireFormatter::VarintSize(MAX_LENGTH) // The varint indicating the actual number of bytes. + + WireFormatter::VarintSize(WireFormatter::MakeTag(field_number, + WireFormatter::WireType::LENGTH_DELIMITED)); // The field and tag comby + } + + //! When serialized with the all elements set, how much bytes are then required. + /*! + This function is used when the field bytes or string is serialized packed. Think in a repeated field. + \return The number of bytes required at most. + */ + static constexpr uint32_t max_serialized_size() + { + return MAX_LENGTH // The number of bytes of the data. + + WireFormatter::VarintSize(MAX_LENGTH); // The varint indicating the actual number of bytes. } protected: diff --git a/src/Fields.h b/src/Fields.h index 4e391f9..653489b 100644 --- a/src/Fields.h +++ b/src/Fields.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2024 Embedded AMS B.V. - All Rights Reserved + * Copyright (C) 2020-2025 Embedded AMS B.V. - All Rights Reserved * * This file is part of Embedded Proto. * @@ -16,7 +16,7 @@ * along with Embedded Proto. If not, see . * * For commercial and closed source application please visit: - * . + * . * * Embedded AMS B.V. * Info: @@ -119,12 +119,12 @@ namespace EmbeddedProto #endif // End of MSG_TO_STRING }; - template + template class FieldTemplate { public: using TYPE = VARIABLE_TYPE; - using CLASS_TYPE = FieldTemplate; + using CLASS_TYPE = FieldTemplate; FieldTemplate() = default; FieldTemplate(const VARIABLE_TYPE& v) : value_(v) { }; @@ -209,18 +209,18 @@ namespace EmbeddedProto bool operator>=(const VARIABLE_TYPE& rhs) { return value_ >= rhs; } bool operator<=(const VARIABLE_TYPE& rhs) { return value_ <= rhs; } - template - bool operator==(const FieldTemplate& rhs) { return value_ == rhs.get(); } - template - bool operator!=(const FieldTemplate& rhs) { return value_ != rhs.get(); } - template - bool operator>(const FieldTemplate& rhs) { return value_ > rhs.get(); } - template - bool operator<(const FieldTemplate& rhs) { return value_ < rhs.get(); } - template - bool operator>=(const FieldTemplate& rhs) { return value_ >= rhs.get(); } - template - bool operator<=(const FieldTemplate& rhs) { return value_ <= rhs.get(); } + template + bool operator==(const FieldTemplate& rhs) { return value_ == rhs.get(); } + template + bool operator!=(const FieldTemplate& rhs) { return value_ != rhs.get(); } + template + bool operator>(const FieldTemplate& rhs) { return value_ > rhs.get(); } + template + bool operator<(const FieldTemplate& rhs) { return value_ < rhs.get(); } + template + bool operator>=(const FieldTemplate& rhs) { return value_ >= rhs.get(); } + template + bool operator<=(const FieldTemplate& rhs) { return value_ <= rhs.get(); } void clear() { value_ = static_cast(0); } @@ -231,6 +231,22 @@ namespace EmbeddedProto return calcBuffer.get_size(); } + //! When serialized with the most unfavrouble value how much bytes does this field need. + /*! + This function takes into account the field number and tag combination. + \param[in] field_number We need to include the field number. This because large field numbers require more bytes. + \return The number of bytes required at most. + */ + static constexpr uint32_t max_serialized_size(const uint32_t field_number) + { + return MAX_SER_SIZE + WireFormatter::VarintSize(WireFormatter::MakeTag(field_number, WIRETYPE)); + } + + static constexpr uint32_t max_serialized_size() + { + return MAX_SER_SIZE; + } + #ifdef MSG_TO_STRING //! Write all the data in this field to a human readable string. @@ -438,22 +454,22 @@ namespace EmbeddedProto }; - using int32 = FieldTemplate; - using int64 = FieldTemplate; - using uint32 = FieldTemplate; - using uint64 = FieldTemplate; - using sint32 = FieldTemplate; - using sint64 = FieldTemplate; - using boolean = FieldTemplate; - using fixed32 = FieldTemplate; - using fixed64 = FieldTemplate; - using sfixed32 = FieldTemplate; - using sfixed64 = FieldTemplate; - using floatfixed = FieldTemplate; - using doublefixed = FieldTemplate; - - template - using enumeration = FieldTemplate; + using int32 = FieldTemplate; + using int64 = FieldTemplate; + using uint32 = FieldTemplate; + using uint64 = FieldTemplate; + using sint32 = FieldTemplate; + using sint64 = FieldTemplate; + using boolean = FieldTemplate; + using fixed32 = FieldTemplate; + using fixed64 = FieldTemplate; + using sfixed32 = FieldTemplate; + using sfixed64 = FieldTemplate; + using floatfixed = FieldTemplate; + using doublefixed = FieldTemplate; + + template + using enumeration = FieldTemplate; } // End of namespace EmbeddedProto. #endif diff --git a/src/ReadBufferFixedSize.h b/src/ReadBufferFixedSize.h index ed38e9e..12867f6 100644 --- a/src/ReadBufferFixedSize.h +++ b/src/ReadBufferFixedSize.h @@ -54,7 +54,7 @@ namespace EmbeddedProto //! \see ::EmbeddedProto::ReadBufferInterface::get_size() uint32_t get_size() const override { - return write_index_; + return write_index_ - read_index_; } //! \see ::EmbeddedProto::ReadBufferInterface::get_max_size() diff --git a/src/RepeatedField.h b/src/RepeatedField.h index d6ea279..bdd26e9 100644 --- a/src/RepeatedField.h +++ b/src/RepeatedField.h @@ -55,20 +55,20 @@ namespace EmbeddedProto struct is_specialization_of_FieldTemplate : std::false_type {}; //! Definition of a trait to check if DATA_TYPE is a specialization of the FieldTemplate. - template - struct is_specialization_of_FieldTemplate<::EmbeddedProto::FieldTemplate> : std::true_type {}; + template + struct is_specialization_of_FieldTemplate<::EmbeddedProto::FieldTemplate> : std::true_type {}; //! This class only supports Field and FieldTemplate classes as template parameter. static_assert(std::is_base_of<::EmbeddedProto::Field, DATA_TYPE>::value || is_specialization_of_FieldTemplate::value, "A Field can only be used as template paramter."); + public: + //! Check how this field shoeld be serialized, packed or not. static constexpr bool REPEATED_FIELD_IS_PACKED = !(std::is_base_of::value || std::is_base_of::value); - public: - RepeatedField() = default; ~RepeatedField() override = default; diff --git a/src/RepeatedFieldFixedSize.h b/src/RepeatedFieldFixedSize.h index 5a00e1c..67b33d7 100644 --- a/src/RepeatedFieldFixedSize.h +++ b/src/RepeatedFieldFixedSize.h @@ -193,6 +193,32 @@ namespace EmbeddedProto //! Return a reference to the internal data storage array. const std::array& get_data_const() const { return data_; } + //! When serialized with the most unfavrouble value how much bytes does this field need. + /*! + This function takes into account the field number and tag combination. + \param[in] field_number We need to include the field number. This because large field numbers require more bytes. + \return The number of bytes required at most. + */ + static constexpr uint32_t max_serialized_size(const uint32_t field_number) + { + return RepeatedField::REPEATED_FIELD_IS_PACKED + ? max_packed_serialized_size(field_number) + : max_unpacked_serialized_size(field_number); + } + + static constexpr uint32_t max_packed_serialized_size(const uint32_t field_number) + { + return WireFormatter::VarintSize(WireFormatter::MakeTag(field_number, + WireFormatter::WireType::LENGTH_DELIMITED)) + + WireFormatter::VarintSize(MAX_LENGTH) + + (MAX_LENGTH * DATA_TYPE::max_serialized_size()); + } + + static constexpr uint32_t max_unpacked_serialized_size(const uint32_t field_number) + { + return MAX_LENGTH * DATA_TYPE::max_serialized_size(field_number); + } + private: //! Number of item in the data array. diff --git a/src/WireFormatter.h b/src/WireFormatter.h index 6db0422..f1c33a2 100644 --- a/src/WireFormatter.h +++ b/src/WireFormatter.h @@ -81,6 +81,24 @@ namespace EmbeddedProto FIXED32 = 5, //!< fixed32, sfixed32, float }; + //! Calculate the number of bytes a varint value will take. + /*! + \param[in] value The value of which to calculate the size. + \return The number of bytes required for serializing the varint. + */ + static constexpr uint32_t VarintSize(const uint64_t value) + { + return (value < (1ULL << 7)) ? 1 : + (value < (1ULL << 14)) ? 2 : + (value < (1ULL << 21)) ? 3 : + (value < (1ULL << 28)) ? 4 : + (value < (1ULL << 35)) ? 5 : + (value < (1ULL << 42)) ? 6 : + (value < (1ULL << 49)) ? 7 : + (value < (1ULL << 56)) ? 8 : + (value < (1ULL << 63)) ? 9 : 10; + } + //! Encode a signed integer using the zig zag method /*! As specified the right-shift must be arithmetic, hence the cast is after the shift. The From d6979524c2b9b73a3fc29c5c87626bb5636a912f Mon Sep 17 00:00:00 2001 From: Benedek Kupper Date: Wed, 12 Nov 2025 10:30:43 +0100 Subject: [PATCH 2/2] ignore generator files created by setup.py --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index da1951e..8d97d07 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ code_coverage_report/** # Ignore the generated options file. generator/EmbeddedProto/embedded_proto_options_pb2.py +generator/EmbeddedProto.egg-info