Skip to content

Commit 21ec7c5

Browse files
authored
Refactor wire format encoder and decoder (#124)
1 parent 1c971b2 commit 21ec7c5

17 files changed

+1060
-1005
lines changed

lib/protobuf/decoder.ex

Lines changed: 143 additions & 440 deletions
Large diffs are not rendered by default.

lib/protobuf/encoder.ex

Lines changed: 8 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
defmodule Protobuf.Encoder do
22
@moduledoc false
3-
import Protobuf.WireTypes
4-
import Bitwise, only: [bsr: 2, band: 2, bsl: 2, bor: 2]
3+
import Protobuf.Wire.Types
4+
import Bitwise, only: [bsl: 2, bor: 2]
55

6-
alias Protobuf.{MessageProps, FieldProps}
6+
alias Protobuf.{FieldProps, MessageProps, Wire, Wire.Varint}
77

88
@spec encode(atom, map | struct, keyword) :: iodata
99
def encode(mod, msg, opts) do
@@ -98,7 +98,7 @@ defmodule Protobuf.Encoder do
9898
@spec encode_field(atom, any, FieldProps.t()) :: iodata
9999
defp encode_field(:normal, val, %{encoded_fnum: fnum, type: type, repeated?: is_repeated}) do
100100
repeated_or_not(val, is_repeated, fn v ->
101-
[fnum | encode_type(type, v)]
101+
[fnum | Wire.from_proto(type, v)]
102102
end)
103103
end
104104

@@ -114,14 +114,14 @@ defmodule Protobuf.Encoder do
114114
# so that oneof {:atom, v} can be encoded
115115
encoded = encode(type, v, iolist: true)
116116
byte_size = IO.iodata_length(encoded)
117-
[fnum | encode_varint(byte_size)] ++ encoded
117+
[fnum | Varint.encode(byte_size)] ++ encoded
118118
end)
119119
end
120120

121121
defp encode_field(:packed, val, %{type: type, encoded_fnum: fnum}) do
122-
encoded = Enum.map(val, fn v -> encode_type(type, v) end)
122+
encoded = Enum.map(val, fn v -> Wire.from_proto(type, v) end)
123123
byte_size = IO.iodata_length(encoded)
124-
[fnum | encode_varint(byte_size)] ++ encoded
124+
[fnum | Varint.encode(byte_size)] ++ encoded
125125
end
126126

127127
@spec class_field(map) :: atom
@@ -143,77 +143,10 @@ defmodule Protobuf.Encoder do
143143
fnum
144144
|> bsl(3)
145145
|> bor(wire_type)
146-
|> encode_varint()
146+
|> Varint.encode()
147147
|> IO.iodata_to_binary()
148148
end
149149

150-
@doc false
151-
@spec encode_type(atom, any) :: iodata
152-
def encode_type(:int32, n) when n >= -0x80000000 and n <= 0x7FFFFFFF, do: encode_varint(n)
153-
154-
def encode_type(:int64, n) when n >= -0x8000000000000000 and n <= 0x7FFFFFFFFFFFFFFF,
155-
do: encode_varint(n)
156-
157-
def encode_type(:string, n), do: encode_type(:bytes, n)
158-
def encode_type(:uint32, n) when n >= 0 and n <= 0xFFFFFFFF, do: encode_varint(n)
159-
def encode_type(:uint64, n) when n >= 0 and n <= 0xFFFFFFFFFFFFFFFF, do: encode_varint(n)
160-
def encode_type(:bool, true), do: encode_varint(1)
161-
def encode_type(:bool, false), do: encode_varint(0)
162-
def encode_type({:enum, type}, n) when is_atom(n), do: n |> type.value() |> encode_varint()
163-
def encode_type({:enum, _}, n), do: encode_varint(n)
164-
def encode_type(:float, :infinity), do: [0, 0, 128, 127]
165-
def encode_type(:float, :negative_infinity), do: [0, 0, 128, 255]
166-
def encode_type(:float, :nan), do: [0, 0, 192, 127]
167-
def encode_type(:float, n), do: <<n::32-float-little>>
168-
def encode_type(:double, :infinity), do: [0, 0, 0, 0, 0, 0, 240, 127]
169-
def encode_type(:double, :negative_infinity), do: [0, 0, 0, 0, 0, 0, 240, 255]
170-
def encode_type(:double, :nan), do: [1, 0, 0, 0, 0, 0, 248, 127]
171-
def encode_type(:double, n), do: <<n::64-float-little>>
172-
173-
def encode_type(:bytes, n) do
174-
len = n |> IO.iodata_length() |> encode_varint()
175-
len ++ n
176-
end
177-
178-
def encode_type(:sint32, n) when n >= -0x80000000 and n <= 0x7FFFFFFF,
179-
do: n |> encode_zigzag |> encode_varint
180-
181-
def encode_type(:sint64, n) when n >= -0x8000000000000000 and n <= 0x7FFFFFFFFFFFFFFF,
182-
do: n |> encode_zigzag |> encode_varint
183-
184-
def encode_type(:fixed64, n) when n >= 0 and n <= 0xFFFFFFFFFFFFFFFF, do: <<n::64-little>>
185-
186-
def encode_type(:sfixed64, n) when n >= -0x8000000000000000 and n <= 0x7FFFFFFFFFFFFFFF,
187-
do: <<n::64-signed-little>>
188-
189-
def encode_type(:fixed32, n) when n >= 0 and n <= 0xFFFFFFFF, do: <<n::32-little>>
190-
191-
def encode_type(:sfixed32, n) when n >= -0x80000000 and n <= 0x7FFFFFFF,
192-
do: <<n::32-signed-little>>
193-
194-
def encode_type(type, n) do
195-
raise Protobuf.TypeEncodeError, message: "#{inspect(n)} is invalid for type #{type}"
196-
end
197-
198-
@spec encode_zigzag(integer) :: integer
199-
defp encode_zigzag(val) when val >= 0, do: val * 2
200-
defp encode_zigzag(val) when val < 0, do: val * -2 - 1
201-
202-
@doc false
203-
@spec encode_varint(integer) :: iolist
204-
def encode_varint(n) when n < 0 do
205-
<<n::64-unsigned-native>> = <<n::64-signed-native>>
206-
encode_varint(n)
207-
end
208-
209-
def encode_varint(n) when n <= 127 do
210-
[n]
211-
end
212-
213-
def encode_varint(n) do
214-
[<<1::1, band(n, 127)::7>> | encode_varint(bsr(n, 7))]
215-
end
216-
217150
@doc false
218151
@spec wire_type(atom) :: integer
219152
def wire_type(:int32), do: wire_varint()

lib/protobuf/wire.ex

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
defmodule Protobuf.Wire do
2+
@moduledoc """
3+
Utilities to convert data from wire format to protobuf and back.
4+
"""
5+
6+
alias Protobuf.Wire.{Varint, Zigzag}
7+
8+
require Logger
9+
10+
@type proto_type ::
11+
:int32
12+
| :int64
13+
| :fixed32
14+
| :fixed64
15+
| :uint32
16+
| :uint64
17+
| :sfixed32
18+
| :sfixed64
19+
| :sint32
20+
| :sint64
21+
| :float
22+
| :double
23+
| :bool
24+
| :string
25+
| :bytes
26+
| {:enum, any}
27+
28+
@type proto_float :: :infinity | :negative_infinity | :nan | float
29+
30+
@type proto_value :: binary | integer | bool | proto_float | atom
31+
32+
@sint32_range -0x80000000..0x7FFFFFFF
33+
@sint64_range -0x8000000000000000..0x7FFFFFFFFFFFFFFF
34+
@uint32_range 0..0xFFFFFFFF
35+
@uint64_range 0..0xFFFFFFFFFFFFFFFF
36+
37+
@spec from_proto(proto_type, proto_value) :: iodata
38+
# Returns improper list, but still valid iodata.
39+
def from_proto(type, binary) when type in [:string, :bytes] do
40+
len = binary |> IO.iodata_length() |> Varint.encode()
41+
len ++ binary
42+
end
43+
44+
def from_proto(:int32, n) when n in @sint32_range, do: Varint.encode(n)
45+
def from_proto(:int64, n) when n in @sint64_range, do: Varint.encode(n)
46+
def from_proto(:uint32, n) when n in @uint32_range, do: Varint.encode(n)
47+
def from_proto(:uint64, n) when n in @uint64_range, do: Varint.encode(n)
48+
49+
def from_proto(:bool, true), do: Varint.encode(1)
50+
def from_proto(:bool, false), do: Varint.encode(0)
51+
52+
def from_proto({:enum, enum}, key) when is_atom(key), do: Varint.encode(enum.value(key))
53+
def from_proto({:enum, _}, n) when is_integer(n), do: Varint.encode(n)
54+
55+
def from_proto(:float, :infinity), do: [0, 0, 128, 127]
56+
def from_proto(:float, :negative_infinity), do: [0, 0, 128, 255]
57+
def from_proto(:float, :nan), do: [0, 0, 192, 127]
58+
def from_proto(:float, n), do: <<n::32-float-little>>
59+
60+
def from_proto(:double, :infinity), do: [0, 0, 0, 0, 0, 0, 240, 127]
61+
def from_proto(:double, :negative_infinity), do: [0, 0, 0, 0, 0, 0, 240, 255]
62+
def from_proto(:double, :nan), do: [1, 0, 0, 0, 0, 0, 248, 127]
63+
def from_proto(:double, n), do: <<n::64-float-little>>
64+
65+
def from_proto(:sint32, n) when n in @sint32_range, do: Varint.encode(Zigzag.encode(n))
66+
def from_proto(:sint64, n) when n in @sint64_range, do: Varint.encode(Zigzag.encode(n))
67+
def from_proto(:fixed32, n) when n in @uint32_range, do: <<n::32-little>>
68+
def from_proto(:fixed64, n) when n in @uint64_range, do: <<n::64-little>>
69+
def from_proto(:sfixed32, n) when n in @sint32_range, do: <<n::32-signed-little>>
70+
def from_proto(:sfixed64, n) when n in @sint64_range, do: <<n::64-signed-little>>
71+
72+
def from_proto(type, n) do
73+
raise Protobuf.TypeEncodeError, message: "#{inspect(n)} is invalid for type #{type}"
74+
end
75+
76+
@spec to_proto(proto_type, binary | integer) :: proto_value
77+
def to_proto(type, val) when type in [:string, :bytes], do: val
78+
79+
def to_proto(:int32, val) do
80+
<<n::signed-integer-32>> = <<val::32>>
81+
n
82+
end
83+
84+
def to_proto(:int64, val) do
85+
<<n::signed-integer-64>> = <<val::64>>
86+
n
87+
end
88+
89+
def to_proto(:uint32, val) do
90+
<<n::unsigned-integer-32>> = <<val::32>>
91+
n
92+
end
93+
94+
def to_proto(:uint64, val) do
95+
<<n::unsigned-integer-64>> = <<val::64>>
96+
n
97+
end
98+
99+
def to_proto(:bool, val), do: val != 0
100+
101+
def to_proto({:enum, enum}, val) do
102+
enum.key(val)
103+
rescue
104+
FunctionClauseError ->
105+
Logger.warn("unknown enum value #{val} when decoding for #{inspect(enum)}")
106+
val
107+
end
108+
109+
def to_proto(:float, <<n::little-float-32>>), do: n
110+
# little endianness, should be 0b0_11111111_000000000...
111+
def to_proto(:float, <<0, 0, 0b1000_0000::8, 0b01111111::8>>), do: :infinity
112+
# little endianness, should be 0b1_11111111_000000000...
113+
def to_proto(:float, <<0, 0, 0b1000_0000::8, 0b11111111::8>>), do: :negative_infinity
114+
# should be 0b*_11111111_not_zero...
115+
def to_proto(:float, <<a::16, 1::1, b::7, _::1, 0b1111111::7>>) when a != 0 or b != 0,
116+
do: :nan
117+
118+
def to_proto(:double, <<n::little-float-64>>), do: n
119+
# little endianness, should be 0b0_11111111111_000000000...
120+
def to_proto(:double, <<0::48, 0b1111::4, 0::4, 0b01111111::8>>), do: :infinity
121+
# little endianness, should be 0b1_11111111111_000000000...
122+
def to_proto(:double, <<0::48, 0b1111::4, 0::4, 0b11111111::8>>), do: :negative_infinity
123+
124+
def to_proto(:double, <<a::48, 0b1111::4, b::4, _::1, 0b1111111::7>>) when a != 0 or b != 0,
125+
do: :nan
126+
127+
def to_proto(type, val) when type in [:sint32, :sint64], do: Zigzag.decode(val)
128+
def to_proto(:fixed32, <<n::little-32>>), do: n
129+
def to_proto(:fixed64, <<n::little-64>>), do: n
130+
def to_proto(:sfixed32, <<n::little-signed-32>>), do: n
131+
def to_proto(:sfixed64, <<n::little-signed-64>>), do: n
132+
133+
def to_proto(type, val) do
134+
raise Protobuf.DecodeError, message: "can't decode #{inspect(val)} into type #{type}"
135+
end
136+
end

lib/protobuf/wire_types.ex renamed to lib/protobuf/wire/types.ex

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
defmodule Protobuf.WireTypes do
1+
defmodule Protobuf.Wire.Types do
22
@moduledoc false
33

44
defmacro wire_varint, do: 0

0 commit comments

Comments
 (0)