Skip to content

Commit 5159e84

Browse files
miguelgrinbergmiguelgrinberg
committed
Support msgpack and custom packet serializers (Fixes #749)
1 parent a813bde commit 5159e84

File tree

10 files changed

+148
-39
lines changed

10 files changed

+148
-39
lines changed

‎src/socketio/asyncio_client.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ async def emit(self, event, data=None, namespace=None, callback=None):
220220
data = [data]
221221
else:
222222
data = []
223-
await self._send_packet(packet.Packet(
223+
await self._send_packet(self.packet_class(
224224
packet.EVENT, namespace=namespace, data=[event] + data, id=id))
225225

226226
async def send(self, data, namespace=None, callback=None):
@@ -296,7 +296,7 @@ async def disconnect(self):
296296
# here we just request the disconnection
297297
# later in _handle_eio_disconnect we invoke the disconnect handler
298298
for n in self.namespaces:
299-
await self._send_packet(packet.Packet(packet.DISCONNECT,
299+
await self._send_packet(self.packet_class(packet.DISCONNECT,
300300
namespace=n))
301301
await self.eio.disconnect(abort=True)
302302

@@ -379,7 +379,7 @@ async def _handle_event(self, namespace, id, data):
379379
data = list(r)
380380
else:
381381
data = [r]
382-
await self._send_packet(packet.Packet(
382+
await self._send_packet(self.packet_class(
383383
packet.ACK, namespace=namespace, id=id, data=data))
384384

385385
async def _handle_ack(self, namespace, id, data):
@@ -482,7 +482,7 @@ async def _handle_eio_connect(self):
482482
self.sid = self.eio.sid
483483
real_auth = await self._get_real_value(self.connection_auth)
484484
for n in self.connection_namespaces:
485-
await self._send_packet(packet.Packet(
485+
await self._send_packet(self.packet_class(
486486
packet.CONNECT, data=real_auth, namespace=n))
487487

488488
async def _handle_eio_message(self, data):
@@ -496,7 +496,7 @@ async def _handle_eio_message(self, data):
496496
else:
497497
await self._handle_ack(pkt.namespace, pkt.id, pkt.data)
498498
else:
499-
pkt = packet.Packet(encoded_packet=data)
499+
pkt = self.packet_class(encoded_packet=data)
500500
if pkt.packet_type == packet.CONNECT:
501501
await self._handle_connect(pkt.namespace, pkt.data)
502502
elif pkt.packet_type == packet.DISCONNECT:

‎src/socketio/asyncio_server.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -369,7 +369,7 @@ async def disconnect(self, sid, namespace=None, ignore_queue=False):
369369
if delete_it:
370370
self.logger.info('Disconnecting %s [%s]', sid, namespace)
371371
eio_sid = self.manager.pre_disconnect(sid, namespace=namespace)
372-
await self._send_packet(eio_sid, packet.Packet(
372+
await self._send_packet(eio_sid, self.packet_class(
373373
packet.DISCONNECT, namespace=namespace))
374374
await self._trigger_event('disconnect', namespace, sid)
375375
self.manager.disconnect(sid, namespace=namespace)
@@ -423,7 +423,7 @@ async def _emit_internal(self, sid, event, data, namespace=None, id=None):
423423
data = [data]
424424
else:
425425
data = []
426-
await self._send_packet(sid, packet.Packet(
426+
await self._send_packet(sid, self.packet_class(
427427
packet.EVENT, namespace=namespace, data=[event] + data, id=id))
428428

429429
async def _send_packet(self, eio_sid, pkt):
@@ -440,7 +440,7 @@ async def _handle_connect(self, eio_sid, namespace, data):
440440
namespace = namespace or '/'
441441
sid = self.manager.connect(eio_sid, namespace)
442442
if self.always_connect:
443-
await self._send_packet(eio_sid, packet.Packet(
443+
await self._send_packet(eio_sid, self.packet_class(
444444
packet.CONNECT, {'sid': sid}, namespace=namespace))
445445
fail_reason = exceptions.ConnectionRefusedError().error_args
446446
try:
@@ -461,15 +461,15 @@ async def _handle_connect(self, eio_sid, namespace, data):
461461
if success is False:
462462
if self.always_connect:
463463
self.manager.pre_disconnect(sid, namespace)
464-
await self._send_packet(eio_sid, packet.Packet(
464+
await self._send_packet(eio_sid, self.packet_class(
465465
packet.DISCONNECT, data=fail_reason, namespace=namespace))
466466
else:
467-
await self._send_packet(eio_sid, packet.Packet(
467+
await self._send_packet(eio_sid, self.packet_class(
468468
packet.CONNECT_ERROR, data=fail_reason,
469469
namespace=namespace))
470470
self.manager.disconnect(sid, namespace)
471471
elif not self.always_connect:
472-
await self._send_packet(eio_sid, packet.Packet(
472+
await self._send_packet(eio_sid, self.packet_class(
473473
packet.CONNECT, {'sid': sid}, namespace=namespace))
474474

475475
async def _handle_disconnect(self, eio_sid, namespace):
@@ -511,7 +511,7 @@ async def _handle_event_internal(self, server, sid, eio_sid, data,
511511
data = list(r)
512512
else:
513513
data = [r]
514-
await server._send_packet(eio_sid, packet.Packet(
514+
await server._send_packet(eio_sid, self.packet_class(
515515
packet.ACK, namespace=namespace, id=id, data=data))
516516

517517
async def _handle_ack(self, eio_sid, namespace, id, data):
@@ -560,7 +560,7 @@ async def _handle_eio_message(self, eio_sid, data):
560560
await self._handle_ack(eio_sid, pkt.namespace, pkt.id,
561561
pkt.data)
562562
else:
563-
pkt = packet.Packet(encoded_packet=data)
563+
pkt = self.packet_class(encoded_packet=data)
564564
if pkt.packet_type == packet.CONNECT:
565565
await self._handle_connect(eio_sid, pkt.namespace, pkt.data)
566566
elif pkt.packet_type == packet.DISCONNECT:

‎src/socketio/client.py‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,13 @@ class Client(object):
5757
use. To disable logging set to ``False``. The default is
5858
``False``. Note that fatal errors are logged even when
5959
``logger`` is ``False``.
60+
:param serializer: The serialization method to use when transmitting
61+
packets. Valid values are ``'default'``, ``'pickle'``,
62+
``'msgpack'`` and ``'cbor'``. Alternatively, a subclass
63+
of the :class:`Packet` class with custom implementations
64+
of the ``encode()`` and ``decode()`` methods can be
65+
provided. Client and server must use compatible
66+
serializers.
6067
:param json: An alternative json module to use for encoding and decoding
6168
packets. Custom json modules must have ``dumps`` and ``loads``
6269
functions that are compatible with the standard library
@@ -82,7 +89,8 @@ class Client(object):
8289
"""
8390
def __init__(self, reconnection=True, reconnection_attempts=0,
8491
reconnection_delay=1, reconnection_delay_max=5,
85-
randomization_factor=0.5, logger=False, json=None, **kwargs):
92+
randomization_factor=0.5, logger=False, serializer='default',
93+
json=None, **kwargs):
8694
global original_signal_handler
8795
if original_signal_handler is None and \
8896
threading.current_thread() == threading.main_thread():
@@ -98,8 +106,15 @@ def __init__(self, reconnection=True, reconnection_attempts=0,
98106
engineio_logger = engineio_options.pop('engineio_logger', None)
99107
if engineio_logger is not None:
100108
engineio_options['logger'] = engineio_logger
109+
if serializer == 'default':
110+
self.packet_class = packet.Packet
111+
elif serializer == 'msgpack':
112+
from . import msgpack_packet
113+
self.packet_class = msgpack_packet.MsgPackPacket
114+
else:
115+
self.packet_class = serializer
101116
if json is not None:
102-
packet.Packet.json = json
117+
self.packet_class.json = json
103118
engineio_options['json'] = json
104119

105120
self.eio = self._engineio_client_class()(**engineio_options)
@@ -381,8 +396,8 @@ def emit(self, event, data=None, namespace=None, callback=None):
381396
data = [data]
382397
else:
383398
data = []
384-
self._send_packet(packet.Packet(packet.EVENT, namespace=namespace,
385-
data=[event] + data, id=id))
399+
self._send_packet(self.packet_class(packet.EVENT, namespace=namespace,
400+
data=[event] + data, id=id))
386401

387402
def send(self, data, namespace=None, callback=None):
388403
"""Send a message to one or more connected clients.
@@ -448,7 +463,8 @@ def disconnect(self):
448463
# here we just request the disconnection
449464
# later in _handle_eio_disconnect we invoke the disconnect handler
450465
for n in self.namespaces:
451-
self._send_packet(packet.Packet(packet.DISCONNECT, namespace=n))
466+
self._send_packet(self.packet_class(
467+
packet.DISCONNECT, namespace=n))
452468
self.eio.disconnect(abort=True)
453469

454470
def get_sid(self, namespace=None):
@@ -557,8 +573,8 @@ def _handle_event(self, namespace, id, data):
557573
data = list(r)
558574
else:
559575
data = [r]
560-
self._send_packet(packet.Packet(packet.ACK, namespace=namespace,
561-
id=id, data=data))
576+
self._send_packet(self.packet_class(
577+
packet.ACK, namespace=namespace, id=id, data=data))
562578

563579
def _handle_ack(self, namespace, id, data):
564580
namespace = namespace or '/'
@@ -647,7 +663,7 @@ def _handle_eio_connect(self):
647663
self.sid = self.eio.sid
648664
real_auth = self._get_real_value(self.connection_auth)
649665
for n in self.connection_namespaces:
650-
self._send_packet(packet.Packet(
666+
self._send_packet(self.packet_class(
651667
packet.CONNECT, data=real_auth, namespace=n))
652668

653669
def _handle_eio_message(self, data):
@@ -661,7 +677,7 @@ def _handle_eio_message(self, data):
661677
else:
662678
self._handle_ack(pkt.namespace, pkt.id, pkt.data)
663679
else:
664-
pkt = packet.Packet(encoded_packet=data)
680+
pkt = self.packet_class(encoded_packet=data)
665681
if pkt.packet_type == packet.CONNECT:
666682
self._handle_connect(pkt.namespace, pkt.data)
667683
elif pkt.packet_type == packet.DISCONNECT:

‎src/socketio/msgpack_packet.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import msgpack
2+
from . import packet
3+
4+
5+
class MsgPackPacket(packet.Packet):
6+
def encode(self):
7+
"""Encode the packet for transmission."""
8+
return msgpack.dumps(self._to_dict())
9+
10+
def decode(self, encoded_packet):
11+
"""Decode a transmitted package."""
12+
decoded = msgpack.loads(encoded_packet)
13+
self.packet_type = decoded['type']
14+
self.data = decoded['data']
15+
self.id = decoded.get('id')
16+
self.namespace = decoded['nsp']

‎src/socketio/packet.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def __init__(self, packet_type=EVENT, data=None, namespace=None, id=None,
3737
self.attachment_count = 0
3838
self.attachments = []
3939
if encoded_packet:
40-
self.attachment_count = self.decode(encoded_packet)
40+
self.attachment_count = self.decode(encoded_packet) or 0
4141

4242
def encode(self):
4343
"""Encode the packet for transmission.
@@ -175,3 +175,13 @@ def _data_is_binary(self, data):
175175
False)
176176
else:
177177
return False
178+
179+
def _to_dict(self):
180+
d = {
181+
'type': self.packet_type,
182+
'data': self.data,
183+
'nsp': self.namespace,
184+
}
185+
if self.id:
186+
d['id'] = self.id
187+
return d

‎src/socketio/server.py‎

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,13 @@ class Server(object):
2424
use. To disable logging set to ``False``. The default is
2525
``False``. Note that fatal errors are logged even when
2626
``logger`` is ``False``.
27+
:param serializer: The serialization method to use when transmitting
28+
packets. Valid values are ``'default'``, ``'pickle'``,
29+
``'msgpack'`` and ``'cbor'``. Alternatively, a subclass
30+
of the :class:`Packet` class with custom implementations
31+
of the ``encode()`` and ``decode()`` methods can be
32+
provided. Client and server must use compatible
33+
serializers.
2734
:param json: An alternative json module to use for encoding and decoding
2835
packets. Custom json modules must have ``dumps`` and ``loads``
2936
functions that are compatible with the standard library
@@ -48,10 +55,11 @@ class Server(object):
4855
4956
:param async_mode: The asynchronous model to use. See the Deployment
5057
section in the documentation for a description of the
51-
available options. Valid async modes are "threading",
52-
"eventlet", "gevent" and "gevent_uwsgi". If this
53-
argument is not given, "eventlet" is tried first, then
54-
"gevent_uwsgi", then "gevent", and finally "threading".
58+
available options. Valid async modes are
59+
``'threading'``, ``'eventlet'``, ``'gevent'`` and
60+
``'gevent_uwsgi'``. If this argument is not given,
61+
``'eventlet'`` is tried first, then ``'gevent_uwsgi'``,
62+
then ``'gevent'``, and finally ``'threading'``.
5563
The first async mode that has all its dependencies
5664
installed is then one that is chosen.
5765
:param ping_interval: The interval in seconds at which the server pings
@@ -98,14 +106,22 @@ class Server(object):
98106
fatal errors are logged even when
99107
``engineio_logger`` is ``False``.
100108
"""
101-
def __init__(self, client_manager=None, logger=False, json=None,
102-
async_handlers=True, always_connect=False, **kwargs):
109+
def __init__(self, client_manager=None, logger=False, serializer='default',
110+
json=None, async_handlers=True, always_connect=False,
111+
**kwargs):
103112
engineio_options = kwargs
104113
engineio_logger = engineio_options.pop('engineio_logger', None)
105114
if engineio_logger is not None:
106115
engineio_options['logger'] = engineio_logger
116+
if serializer == 'default':
117+
self.packet_class = packet.Packet
118+
elif serializer == 'msgpack':
119+
from . import msgpack_packet
120+
self.packet_class = msgpack_packet.MsgPackPacket
121+
else:
122+
self.packet_class = serializer
107123
if json is not None:
108-
packet.Packet.json = json
124+
self.packet_class.json = json
109125
engineio_options['json'] = json
110126
engineio_options['async_handlers'] = False
111127
self.eio = self._engineio_server_class()(**engineio_options)
@@ -531,7 +547,7 @@ def disconnect(self, sid, namespace=None, ignore_queue=False):
531547
if delete_it:
532548
self.logger.info('Disconnecting %s [%s]', sid, namespace)
533549
eio_sid = self.manager.pre_disconnect(sid, namespace=namespace)
534-
self._send_packet(eio_sid, packet.Packet(
550+
self._send_packet(eio_sid, self.packet_class(
535551
packet.DISCONNECT, namespace=namespace))
536552
self._trigger_event('disconnect', namespace, sid)
537553
self.manager.disconnect(sid, namespace=namespace)
@@ -609,7 +625,7 @@ def _emit_internal(self, eio_sid, event, data, namespace=None, id=None):
609625
data = [data]
610626
else:
611627
data = []
612-
self._send_packet(eio_sid, packet.Packet(
628+
self._send_packet(eio_sid, self.packet_class(
613629
packet.EVENT, namespace=namespace, data=[event] + data, id=id))
614630

615631
def _send_packet(self, eio_sid, pkt):
@@ -626,7 +642,7 @@ def _handle_connect(self, eio_sid, namespace, data):
626642
namespace = namespace or '/'
627643
sid = self.manager.connect(eio_sid, namespace)
628644
if self.always_connect:
629-
self._send_packet(eio_sid, packet.Packet(
645+
self._send_packet(eio_sid, self.packet_class(
630646
packet.CONNECT, {'sid': sid}, namespace=namespace))
631647
fail_reason = exceptions.ConnectionRefusedError().error_args
632648
try:
@@ -647,15 +663,15 @@ def _handle_connect(self, eio_sid, namespace, data):
647663
if success is False:
648664
if self.always_connect:
649665
self.manager.pre_disconnect(sid, namespace)
650-
self._send_packet(eio_sid, packet.Packet(
666+
self._send_packet(eio_sid, self.packet_class(
651667
packet.DISCONNECT, data=fail_reason, namespace=namespace))
652668
else:
653-
self._send_packet(eio_sid, packet.Packet(
669+
self._send_packet(eio_sid, self.packet_class(
654670
packet.CONNECT_ERROR, data=fail_reason,
655671
namespace=namespace))
656672
self.manager.disconnect(sid, namespace)
657673
elif not self.always_connect:
658-
self._send_packet(eio_sid, packet.Packet(
674+
self._send_packet(eio_sid, self.packet_class(
659675
packet.CONNECT, {'sid': sid}, namespace=namespace))
660676

661677
def _handle_disconnect(self, eio_sid, namespace):
@@ -697,7 +713,7 @@ def _handle_event_internal(self, server, sid, eio_sid, data, namespace,
697713
data = list(r)
698714
else:
699715
data = [r]
700-
server._send_packet(eio_sid, packet.Packet(
716+
server._send_packet(eio_sid, self.packet_class(
701717
packet.ACK, namespace=namespace, id=id, data=data))
702718

703719
def _handle_ack(self, eio_sid, namespace, id, data):
@@ -737,7 +753,7 @@ def _handle_eio_message(self, eio_sid, data):
737753
else:
738754
self._handle_ack(eio_sid, pkt.namespace, pkt.id, pkt.data)
739755
else:
740-
pkt = packet.Packet(encoded_packet=data)
756+
pkt = self.packet_class(encoded_packet=data)
741757
if pkt.packet_type == packet.CONNECT:
742758
self._handle_connect(eio_sid, pkt.namespace, pkt.data)
743759
elif pkt.packet_type == packet.DISCONNECT:

‎tests/common/test_client.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from socketio import asyncio_namespace
1212
from socketio import client
1313
from socketio import exceptions
14+
from socketio import msgpack_packet
1415
from socketio import namespace
1516
from socketio import packet
1617

@@ -49,8 +50,20 @@ def test_create(self, engineio_client_class):
4950
assert c.callbacks == {}
5051
assert c._binary_packet is None
5152
assert c._reconnect_task is None
53+
assert c.packet_class == packet.Packet
5254

53-
def test_custon_json(self):
55+
def test_msgpack(self):
56+
c = client.Client(serializer='msgpack')
57+
assert c.packet_class == msgpack_packet.MsgPackPacket
58+
59+
def test_custom_serializer(self):
60+
class CustomPacket(packet.Packet):
61+
pass
62+
63+
c = client.Client(serializer=CustomPacket)
64+
assert c.packet_class == CustomPacket
65+
66+
def test_custom_json(self):
5467
client.Client()
5568
assert packet.Packet.json == json
5669
assert engineio_packet.Packet.json == json

0 commit comments

Comments
 (0)