added json schema (#88)

* added json schema

* update readme

* updated version number

Author: mattclarke

README.md

@@ -6,30 +6,31 @@ https://github.com/ess-dmsc/streaming-data-types
 
 ## FlatBuffer Schemas
 
-|name|description|
-|----|-----------|
-|hs00|Histogram schema (deprecated in favour of hs01)|
-|hs01|Histogram schema|
-|ns10|NICOS cache entry schema|
-|pl72|Run start|
-|6s4t|Run stop|
-|f142|Log data (deprecated in favour of f144)|
-|f144|Log data|
-|ev42|Event data (deprecated in favour of ev44)|
-|ev43|Event data from multiple pulses|
-|ev44|Event data with signed data types|
-|x5f2|Status messages|
-|tdct|Timestamps|
-|ep00|EPICS connection info (deprecated in favour of ep01)|
-|ep01|EPICS connection info|
-|rf5k|Forwarder configuration update|
-|answ|File-writer command response|
-|wrdn|File-writer finished writing|
-|NDAr|**Deprecated**|
-|ADAr|EPICS areaDetector data|
-|al00|Alarm/status messages used by the Forwarder and NICOS|
-|senv|**Deprecated**|
-|se00|Arrays with optional timestamps, for example waveform data. Replaces _senv_. |
+| name | description                                                                  |
+|------|------------------------------------------------------------------------------|
+| hs00 | Histogram schema (deprecated in favour of hs01)                              |
+| hs01 | Histogram schema                                                             |
+| ns10 | NICOS cache entry schema                                                     |
+| pl72 | Run start                                                                    |
+| 6s4t | Run stop                                                                     |
+| f142 | Log data (deprecated in favour of f144)                                      |
+| f144 | Log data                                                                     |
+| ev42 | Event data (deprecated in favour of ev44)                                    |
+| ev43 | Event data from multiple pulses                                              |
+| ev44 | Event data with signed data types                                            |
+| x5f2 | Status messages                                                              |
+| tdct | Timestamps                                                                   |
+| ep00 | EPICS connection info (deprecated in favour of ep01)                         |
+| ep01 | EPICS connection info                                                        |
+| rf5k | Forwarder configuration update                                               |
+| answ | File-writer command response                                                 |
+| wrdn | File-writer finished writing                                                 |
+| NDAr | **Deprecated**                                                               |
+| ADAr | EPICS areaDetector data                                                      |
+| al00 | Alarm/status messages used by the Forwarder and NICOS                        |
+| senv | **Deprecated**                                                               |
+| json | Generic JSON data                                                            |
+| se00 | Arrays with optional timestamps, for example waveform data. Replaces _senv_. |
 
 ### hs00 and hs01
 Schema for histogram data. It is one of the more complicated to use schemas.

README_DEV.md

@@ -25,7 +25,6 @@ This command can also be used to run the hooks manually.
 * Check whether the serialised data produced by the new code can be verified in C++
   * There is a helper program in the [FlatBufferVerification](https://github.com/ess-dmsc/FlatBufferVerification) repository
   * Don't worry if it fails verification - it seems to be an inherent FlatBuffers issue
-* Add the schema and verifiability result to the table of schemas in `README.md`
 
 ### Tox
 Tox allows the unit tests to be run against multiple versions of Python.

streaming_data_types/__init__.py

@@ -19,6 +19,7 @@
 )
 from streaming_data_types.histogram_hs00 import deserialise_hs00, serialise_hs00
 from streaming_data_types.histogram_hs01 import deserialise_hs01, serialise_hs01
+from streaming_data_types.json_json import deserialise_json, serialise_json
 from streaming_data_types.logdata_f142 import deserialise_f142, serialise_f142
 from streaming_data_types.logdata_f144 import deserialise_f144, serialise_f144
 from streaming_data_types.nicos_cache_ns10 import deserialise_ns10, serialise_ns10
@@ -56,6 +57,7 @@
     "se00": serialise_se00,
     "ADAr": serialise_ADAr,
     "al00": serialise_al00,
+    "json": serialise_json,
 }
 
 
@@ -82,4 +84,5 @@
     "se00": deserialise_se00,
     "ADAr": deserialise_ADAr,
     "al00": deserialise_al00,
+    "json": deserialise_json,
 }

streaming_data_types/_version.py

@@ -1,4 +1,4 @@
 # Version is not directly defined in __init__ because that causes all
 # run time dependencies to become build-time dependencies when it is
 # imported in setup.py
-version = "0.21.1"
+version = "0.22.0"

streaming_data_types/fbschemas/json_json/JsonData.py

@@ -0,0 +1,67 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace:
+
+import flatbuffers
+from flatbuffers.compat import import_numpy
+
+np = import_numpy()
+
+
+class JsonData(object):
+    __slots__ = ["_tab"]
+
+    @classmethod
+    def GetRootAs(cls, buf, offset=0):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = JsonData()
+        x.Init(buf, n + offset)
+        return x
+
+    @classmethod
+    def GetRootAsJsonData(cls, buf, offset=0):
+        """This method is deprecated. Please switch to GetRootAs."""
+        return cls.GetRootAs(buf, offset)
+
+    @classmethod
+    def JsonDataBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(
+            buf, offset, b"\x6A\x73\x6F\x6E", size_prefixed=size_prefixed
+        )
+
+    # JsonData
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # JsonData
+    def Json(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+
+def JsonDataStart(builder):
+    builder.StartObject(1)
+
+
+def Start(builder):
+    JsonDataStart(builder)
+
+
+def JsonDataAddJson(builder, json):
+    builder.PrependUOffsetTRelativeSlot(
+        0, flatbuffers.number_types.UOffsetTFlags.py_type(json), 0
+    )
+
+
+def AddJson(builder: flatbuffers.Builder, json: int):
+    JsonDataAddJson(builder, json)
+
+
+def JsonDataEnd(builder):
+    return builder.EndObject()
+
+
+def End(builder):
+    return JsonDataEnd(builder)

streaming_data_types/json_json.py

@@ -0,0 +1,24 @@
+import flatbuffers
+
+import streaming_data_types.fbschemas.json_json.JsonData as JsonData
+from streaming_data_types.utils import check_schema_identifier
+
+FILE_IDENTIFIER = b"json"
+
+
+def deserialise_json(buffer) -> str:
+    check_schema_identifier(buffer, FILE_IDENTIFIER)
+    return JsonData.JsonData.GetRootAsJsonData(buffer, 0).Json().decode("utf-8")
+
+
+def serialise_json(json_str) -> bytes:
+    builder = flatbuffers.Builder(128)
+
+    offset = builder.CreateString(json_str)
+
+    JsonData.JsonDataStart(builder)
+    JsonData.AddJson(builder, offset)
+    result = JsonData.JsonDataEnd(builder)
+
+    builder.Finish(result, file_identifier=FILE_IDENTIFIER)
+    return bytes(builder.Output())

tests/test_json.py

@@ -0,0 +1,34 @@
+import json
+
+import pytest
+
+from streaming_data_types import DESERIALISERS, SERIALISERS
+from streaming_data_types.exceptions import WrongSchemaException
+from streaming_data_types.json_json import deserialise_json, serialise_json
+
+
+class TestSerialisationJson:
+    def test_serialises_and_deserialises_json_message_correctly(self):
+        """
+        Round-trip to check what we serialise is what we get back.
+        """
+        json_str = json.dumps(["foo", "bar"])
+        buf = serialise_json(json_str)
+        entry = deserialise_json(buf)
+
+        assert entry == json_str
+
+    def test_if_buffer_has_wrong_id_then_throws(self):
+        json_str = json.dumps(["foo", "bar"])
+        buf = serialise_json(json_str)
+
+        # Manually hack the id
+        buf = bytearray(buf)
+        buf[4:8] = b"1234"
+
+        with pytest.raises(WrongSchemaException):
+            deserialise_json(buf)
+
+    def test_schema_type_is_in_global_serialisers_list(self):
+        assert "json" in SERIALISERS
+        assert "json" in DESERIALISERS