diff --git a/lib/protobuf/encoder.ex b/lib/protobuf/encoder.ex index dd333335..80f7cc74 100644 --- a/lib/protobuf/encoder.ex +++ b/lib/protobuf/encoder.ex @@ -77,35 +77,22 @@ defmodule Protobuf.Encoder do "Got error when encoding #{inspect(struct_mod)}##{prop.name_atom}: #{Exception.format(:error, error)}" end - defp skip_field?(_syntax, [], _prop), do: true - defp skip_field?(_syntax, val, _prop) when is_map(val), do: map_size(val) == 0 - defp skip_field?(:proto2, nil, %FieldProps{optional?: optional?}), do: optional? - defp skip_field?(:proto2, value, %FieldProps{default: value, oneof: nil}), do: true - - defp skip_field?(:proto3, val, %FieldProps{proto3_optional?: true}), - do: is_nil(val) - - defp skip_field?(:proto3, nil, _prop), do: true - defp skip_field?(:proto3, 0, %FieldProps{oneof: nil}), do: true - defp skip_field?(:proto3, +0.0, %FieldProps{oneof: nil}), do: true - defp skip_field?(:proto3, "", %FieldProps{oneof: nil}), do: true - defp skip_field?(:proto3, false, %FieldProps{oneof: nil}), do: true - - defp skip_field?(:proto3, value, %FieldProps{type: {:enum, enum_mod}, oneof: nil}) do - enum_props = enum_mod.__message_props__() - %{name_atom: name_atom, name: name} = enum_props.field_props[0] - value == name_atom or value == name + defp skip_field?(syntax, value, field_prop) do + case Protobuf.Presence.get_field_presence(syntax, value, field_prop) do + :present -> false + # Proto2 required isn't skipped even if not present + :maybe -> not(syntax == :proto2 && field_prop.required?) + :not_present -> not(syntax == :proto2 && field_prop.required?) + end end - defp skip_field?(_syntax, _val, _prop), do: false - defp do_encode_field( :normal, val, syntax, %FieldProps{encoded_fnum: fnum, type: type, repeated?: repeated?} = prop ) do - if skip_field?(syntax, val, prop) or skip_enum?(syntax, val, prop) do + if skip_field?(syntax, val, prop) do :skip else iodata = apply_or_map(val, repeated?, &[fnum | Wire.encode(type, &1)]) @@ -143,7 +130,7 @@ defmodule Protobuf.Encoder do end defp do_encode_field(:packed, val, syntax, %FieldProps{type: type, encoded_fnum: fnum} = prop) do - if skip_field?(syntax, val, prop) or skip_enum?(syntax, val, prop) do + if skip_field?(syntax, val, prop) do :skip else encoded = Enum.map(val, &Wire.encode(type, &1)) @@ -198,20 +185,6 @@ defmodule Protobuf.Encoder do defp apply_or_map(val, _repeated? = true, func), do: Enum.map(val, func) defp apply_or_map(val, _repeated? = false, func), do: func.(val) - defp skip_enum?(:proto2, _value, _prop), do: false - defp skip_enum?(:proto3, _value, %FieldProps{proto3_optional?: true}), do: false - defp skip_enum?(_syntax, _value, %FieldProps{enum?: false}), do: false - - defp skip_enum?(_syntax, _value, %FieldProps{enum?: true, oneof: oneof}) when not is_nil(oneof), - do: false - - defp skip_enum?(_syntax, _value, %FieldProps{required?: true}), do: false - defp skip_enum?(_syntax, value, %FieldProps{type: type}), do: enum_default?(type, value) - - defp enum_default?({:enum, enum_mod}, val) when is_atom(val), do: enum_mod.value(val) == 0 - defp enum_default?({:enum, _enum_mod}, val) when is_integer(val), do: val == 0 - defp enum_default?({:enum, _enum_mod}, list) when is_list(list), do: false - # Returns a map of %{field_name => field_value} from oneofs. For example, if you have: # oneof body { # string a = 1; diff --git a/lib/protobuf/json/encode.ex b/lib/protobuf/json/encode.ex index 13e97e0c..ad031c9f 100644 --- a/lib/protobuf/json/encode.ex +++ b/lib/protobuf/json/encode.ex @@ -162,7 +162,7 @@ defmodule Protobuf.JSON.Encode do defp encode_regular_fields(struct, %{field_props: field_props, syntax: syntax}, opts) do for {_field_num, %{name_atom: name, oneof: nil} = prop} <- field_props, %{^name => value} = struct, - emit?(syntax, prop, value) || opts[:emit_unpopulated] do + emit?(syntax, prop, value, opts[:emit_unpopulated]) do encode_field(prop, value, opts) end end @@ -301,18 +301,17 @@ defmodule Protobuf.JSON.Encode do defp maybe_repeat(%{repeated?: false}, val, fun), do: fun.(val) defp maybe_repeat(%{repeated?: true}, val, fun), do: Enum.map(val, fun) - defp emit?(:proto2, %{default: value}, value), do: false - defp emit?(:proto2, %{optional?: true}, val), do: not is_nil(val) - defp emit?(:proto3, %{proto3_optional?: true}, val), do: not is_nil(val) - defp emit?(_syntax, _prop, +0.0), do: false - defp emit?(_syntax, _prop, nil), do: false - defp emit?(_syntax, _prop, 0), do: false - defp emit?(_syntax, _prop, false), do: false - defp emit?(_syntax, _prop, []), do: false - defp emit?(_syntax, _prop, ""), do: false - defp emit?(_syntax, _prop, %{} = map) when map_size(map) == 0, do: false - defp emit?(_syntax, %{type: {:enum, enum}}, key) when is_atom(key), do: enum.value(key) != 0 - defp emit?(_syntax, _prop, _value), do: true + defp emit?(_syntax, _field_prop, _value, true = _emit_unpopulated?) do + true + end + + defp emit?(syntax, field_prop, value, _emit_unpopulated?) do + case Protobuf.Presence.get_field_presence(syntax, value, field_prop) do + :present -> true + :maybe -> false + :not_present -> false + end + end defp transform_module(message, module) do if transform_module = module.transform_module() do diff --git a/lib/protobuf/presence.ex b/lib/protobuf/presence.ex new file mode 100644 index 00000000..409845bf --- /dev/null +++ b/lib/protobuf/presence.ex @@ -0,0 +1,179 @@ +defmodule Protobuf.Presence do + @moduledoc """ + Helpers for determining Protobuf field presence. + """ + + alias Protobuf.FieldProps + + @doc """ + Returns whether a field or oneof is present, not present, or maybe present + + `:present` and `:not present` mean that a field is **explicitly** present or not, + respectively. + + Some values may be implicitly present. For example, lists in `repeated` fields + always have implicit presence. In these cases, if the presence is ambiguous, + returns `:maybe`. + + For more information about field presence tracking rules, refer to the official + [Field Presence docs](https://protobuf.dev/programming-guides/field_presence/). + + + ## Examples + + # Non-optional proto3 field: + Protobuf.Presence(%MyMessage{foo: 42}, :foo) + #=> :present + + Protobuf.Presence(%MyMessage{foo: 0}, :foo) + #=> :maybe + + Protobuf.Presence(%MyMessage{}, :foo) + #=> :maybe + + # Optional proto3 field: + Protobuf.Presence(%MyMessage{bar: 42}, :bar) + #=> :present + + Protobuf.Presence(%MyMessage{bar: 0}, :bar) + #=> :present + + Protobuf.Presence(%MyMessage{}, :bar) + #=> :not_present + + """ + @spec field_presence(message :: struct(), field :: atom()) :: :present | :not_present | :maybe + def field_presence(%mod{} = message, field) do + message_props = mod.__message_props__() + transformed_message = transform_module(message, mod) + fnum = Map.fetch!(message_props.field_tags, field) + field_prop = Map.fetch!(message_props.field_props, fnum) + value = get_oneof_value(transformed_message, message_props, field, field_prop) + + transformed_value = + case field_prop do + %{embedded: true, type: mod} -> transform_module(value, mod) + _ -> value + end + + get_field_presence(message_props.syntax, transformed_value, field_prop) + end + + defp get_oneof_value(message, message_props, field, field_prop) do + case field_prop.oneof do + nil -> + Map.fetch!(message, field) + + oneof_num -> + {oneof_field, _} = Enum.find(message_props.oneof, fn {_name, tag} -> tag == oneof_num end) + + case Map.fetch!(message, oneof_field) do + {^field, value} -> value + _ -> nil + end + end + end + + defp transform_module(message, module) do + if transform_module = module.transform_module() do + transform_module.encode(message, module) + else + message + end + end + + # We probably want to make this public eventually, but it makes sense to hold + # it until we add editions support, since we definitely don't want to add + # `syntax` in a public API + @doc false + @spec get_field_presence(:proto2 | :proto3, term(), FieldProps.t()) :: :present | :not_present | :maybe + def get_field_presence(syntax, value, field_prop) + + # Repeated and maps are always implicit. + def get_field_presence(_syntax, [], _prop) do + :maybe + end + + def get_field_presence(_syntax, val, _prop) when is_map(val) do + if map_size(val) == 0 do + :maybe + else + :present + end + end + + # For proto2 singular cardinality fields: + # + # - Non-one_of fields with default values have implicit presence + # - Others have explicit presence + def get_field_presence(:proto2, nil, _prop) do + :not_present + end + + def get_field_presence(:proto2, value, %FieldProps{default: value, oneof: nil}) do + :maybe + end + + def get_field_presence(:proto2, _value, _props) do + :present + end + + # For proto3 singular cardinality fields: + # + # - Optional and Oneof fields have explicit presence tracking + # - Other fields have implicit presence tracking + def get_field_presence(:proto3, nil, %FieldProps{proto3_optional?: true}) do + :not_present + end + + def get_field_presence(:proto3, _, %FieldProps{proto3_optional?: true}) do + :present + end + + def get_field_presence(_syntax, value, %FieldProps{oneof: oneof}) when not is_nil(oneof) do + if is_nil(value) do + :not_present + else + :present + end + end + + # Messages have explicit presence tracking in proto3 + def get_field_presence(:proto3, nil, _prop) do + :not_present + end + + # Defaults for different field types: implicit presence means they are maybe set + def get_field_presence(:proto3, 0, _prop) do + :maybe + end + + def get_field_presence(:proto3, +0.0, _prop) do + :maybe + end + + def get_field_presence(:proto3, "", _prop) do + :maybe + end + + def get_field_presence(:proto3, false, _prop) do + :maybe + end + + def get_field_presence(_syntax, value, %FieldProps{type: {:enum, enum_mod}}) do + if enum_default?(enum_mod, value) do + :maybe + else + :present + end + end + + # Finally, everything else. + def get_field_presence(_syntax, _val, _prop) do + :present + end + + defp enum_default?(enum_mod, val) when is_atom(val), do: enum_mod.value(val) == 0 + defp enum_default?(_enum_mod, val) when is_integer(val), do: val == 0 + defp enum_default?(_enum_mod, list) when is_list(list), do: false +end diff --git a/lib/protobuf/text.ex b/lib/protobuf/text.ex index 13e86664..104c2c44 100644 --- a/lib/protobuf/text.ex +++ b/lib/protobuf/text.ex @@ -176,28 +176,14 @@ defmodule Protobuf.Text do end end - # Copied from Protobuf.Encoder. Should it be shared? - defp skip_field?(_syntax, [], _prop), do: true - defp skip_field?(_syntax, val, _prop) when is_map(val), do: map_size(val) == 0 - defp skip_field?(:proto2, nil, %FieldProps{optional?: optional?}), do: optional? - defp skip_field?(:proto2, value, %FieldProps{default: value, oneof: nil}), do: true - defp skip_field?(:proto3, val, %FieldProps{proto3_optional?: true}), do: is_nil(val) - - defp skip_field?(:proto3, nil, _prop), do: true - defp skip_field?(:proto3, 0, %FieldProps{oneof: nil}), do: true - defp skip_field?(:proto3, +0.0, %FieldProps{oneof: nil}), do: true - defp skip_field?(:proto3, "", %FieldProps{oneof: nil}), do: true - defp skip_field?(:proto3, false, %FieldProps{oneof: nil}), do: true - - # This is actually new. Should it be ported to Protobuf.Encoder? - defp skip_field?(:proto3, value, %FieldProps{type: {:enum, enum_mod}, oneof: nil}) do - enum_props = enum_mod.__message_props__() - %{name_atom: name_atom, name: name, json_name: name_json} = enum_props.field_props[0] - - value == name_atom or value == name or value == name_json + defp skip_field?(syntax, value, field_prop) do + case Protobuf.Presence.get_field_presence(syntax, value, field_prop) do + :present -> false + # Proto2 required isn't skipped even if not present + :maybe -> not(syntax == :proto2 && field_prop.required?) + :not_present -> not(syntax == :proto2 && field_prop.required?) + end end - defp skip_field?(_, _, _), do: false - defp inspect_opts(), do: %Inspect.Opts{limit: :infinity} end diff --git a/test/protobuf/presence_test.exs b/test/protobuf/presence_test.exs new file mode 100644 index 00000000..ef43dd51 --- /dev/null +++ b/test/protobuf/presence_test.exs @@ -0,0 +1,146 @@ +defmodule Protobuf.PresenceTest do + use ExUnit.Case, async: true + + alias Protobuf.Presence + alias TestMsg.{Foo, Foo2, Proto3Optional, Oneof, OneofProto3, ContainsTransformModule} + + describe "field_presence/2 for proto3" do + test "singular non-optional fields have implicit presence" do + msg = %Foo{} + assert Presence.field_presence(msg, :a) == :maybe + assert Presence.field_presence(msg, :c) == :maybe + assert Presence.field_presence(msg, :k) == :maybe + + msg = %Foo{a: 42, c: "hello", k: true, j: :A} + assert Presence.field_presence(msg, :a) == :present + assert Presence.field_presence(msg, :c) == :present + assert Presence.field_presence(msg, :k) == :present + assert Presence.field_presence(msg, :j) == :present + + msg = %Foo{a: 0, c: "", k: false, j: :UNKNOWN} + assert Presence.field_presence(msg, :a) == :maybe + assert Presence.field_presence(msg, :c) == :maybe + assert Presence.field_presence(msg, :k) == :maybe + assert Presence.field_presence(msg, :j) == :maybe + end + + test "optional fields have explicit presence" do + msg = %Proto3Optional{} + assert Presence.field_presence(msg, :a) == :not_present + assert Presence.field_presence(msg, :c) == :not_present + + msg = %Proto3Optional{a: 50, c: "hello"} + assert Presence.field_presence(msg, :a) == :present + assert Presence.field_presence(msg, :c) == :present + + msg = %Proto3Optional{a: 0, c: ""} + assert Presence.field_presence(msg, :a) == :present + assert Presence.field_presence(msg, :c) == :present + end + + test "oneof fields have explicit presence" do + msg = %OneofProto3{} + assert Presence.field_presence(msg, :a) == :not_present + assert Presence.field_presence(msg, :b) == :not_present + + msg = %OneofProto3{first: {:a, 42}} + assert Presence.field_presence(msg, :a) == :present + assert Presence.field_presence(msg, :b) == :not_present + + msg = %OneofProto3{first: {:a, 0}} + assert Presence.field_presence(msg, :a) == :present + + msg = %OneofProto3{first: {:e, :UNKNOWN}} + assert Presence.field_presence(msg, :e) == :present + end + + test "message fields have explicit presence" do + msg = %Foo{} + assert Presence.field_presence(msg, :e) == :not_present + + msg = %Foo{e: %Foo.Bar{}} + assert Presence.field_presence(msg, :e) == :present + end + end + + describe "field_presence/2 for proto2" do + test "singular fields have explicit presence" do + msg = %Foo2{} + assert Presence.field_presence(msg, :a) == :not_present + assert Presence.field_presence(msg, :c) == :not_present + assert Presence.field_presence(msg, :k) == :not_present + + msg = %Foo2{a: 42, c: "hello", k: true, j: :A} + assert Presence.field_presence(msg, :a) == :present + assert Presence.field_presence(msg, :c) == :present + assert Presence.field_presence(msg, :k) == :present + assert Presence.field_presence(msg, :j) == :present + + msg = %Foo2{a: 0, c: "", k: false, j: :UNKNOWN} + assert Presence.field_presence(msg, :a) == :present + assert Presence.field_presence(msg, :c) == :present + assert Presence.field_presence(msg, :k) == :present + assert Presence.field_presence(msg, :j) == :present + end + + test "singular fields with default have implicit presence" do + msg = %Foo2{} + assert Presence.field_presence(msg, :b) == :maybe + + msg = %Foo2{b: 5} # 5 is the default value for :b + assert Presence.field_presence(msg, :b) == :maybe + + msg = %Foo2{b: 6} + assert Presence.field_presence(msg, :b) == :present + end + + test "oneof fields have explicit presence" do + msg = %Oneof{} + assert Presence.field_presence(msg, :a) == :not_present + assert Presence.field_presence(msg, :b) == :not_present + + msg = %Oneof{first: {:a, 42}} + assert Presence.field_presence(msg, :a) == :present + assert Presence.field_presence(msg, :b) == :not_present + + # Even if the value is default, it is present + msg = %Oneof{first: {:a, 0}} + assert Presence.field_presence(msg, :a) == :present + + msg = %Oneof{first: {:e, :UNKNOWN}} + assert Presence.field_presence(msg, :e) == :present + end + + test "message fields have explicit presence" do + msg = %Foo2{} + assert Presence.field_presence(msg, :e) == :not_present + + msg = %Foo2{e: %Foo.Bar{}} + assert Presence.field_presence(msg, :e) == :present + end + end + + describe "field_presence/2" do + test "repeated fields have implicit presence" do + msg = %Foo{g: []} + assert Presence.field_presence(msg, :g) == :maybe + + msg = %Foo{g: [1, 2, 3]} + assert Presence.field_presence(msg, :g) == :present + end + + test "maps have implicit presence" do + msg = %Foo{l: %{}} + assert Presence.field_presence(msg, :l) == :maybe + + msg = %Foo{l: %{"key" => 123}} + assert Presence.field_presence(msg, :l) == :present + end + + # Transform module tests + test "field_presence works with transform modules" do + msg = %ContainsTransformModule{field: 42} + assert Presence.field_presence(msg, :field) == :present + end + end +end diff --git a/test/protobuf/text_test.exs b/test/protobuf/text_test.exs index 7b84852e..7fec38ed 100644 --- a/test/protobuf/text_test.exs +++ b/test/protobuf/text_test.exs @@ -143,7 +143,7 @@ defmodule Protobuf.TextTest do end test "raises on absent proto2 required" do - assert_raise RuntimeError, "field :a is required", fn -> + assert_raise Protobuf.EncodeError, "field :a is required", fn -> Text.encode(%TestMsg.Foo2{}) end end diff --git a/test/support/test_msg.ex b/test/support/test_msg.ex index e90b77b3..6a911495 100644 --- a/test/support/test_msg.ex +++ b/test/support/test_msg.ex @@ -82,8 +82,8 @@ defmodule TestMsg do field :g, 8, repeated: true, type: :int32 # field :h, 9, repeated: true, type: Foo.Bar field :i, 10, repeated: true, type: :int32, packed: true - # field :j, 11, optional: true, type: EnumFoo, enum: true - # field :k, 12, optional: true, type: :bool + field :j, 11, optional: true, type: EnumFoo, enum: true + field :k, 12, optional: true, type: :bool field :l, 13, repeated: true, type: MapFoo, map: true field :non_matched, 101, type: :int32, optional: true end