Skip to content

Commit 2123654

Browse files
mattclarkemattclarke
authored
Added ad00 as a replacement for ADAR (#94)
* first attempt for ad00 * bumped version number
1 parent 43c9415 commit 2123654

File tree

8 files changed

+742
-1
lines changed

8 files changed

+742
-1
lines changed

‎streaming_data_types/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from streaming_data_types._version import version
22
from streaming_data_types.action_response_answ import deserialise_answ, serialise_answ
33
from streaming_data_types.alarm_al00 import deserialise_al00, serialise_al00
4+
from streaming_data_types.area_detector_ad00 import deserialise_ad00, serialise_ad00
45
from streaming_data_types.area_detector_ADAr import deserialise_ADAr, serialise_ADAr
56
from streaming_data_types.area_detector_NDAr import deserialise_ndar, serialise_ndar
67
from streaming_data_types.array_1d_se00 import deserialise_se00, serialise_se00
@@ -58,6 +59,7 @@
5859
"ADAr": serialise_ADAr,
5960
"al00": serialise_al00,
6061
"json": serialise_json,
62+
"ad00": serialise_ad00,
6163
}
6264

6365

@@ -85,4 +87,5 @@
8587
"ADAr": deserialise_ADAr,
8688
"al00": deserialise_al00,
8789
"json": deserialise_json,
90+
"ad00": deserialise_ad00,
8891
}

‎streaming_data_types/_version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
# Version is not directly defined in __init__ because that causes all
22
# run time dependencies to become build-time dependencies when it is
33
# imported in setup.py
4-
version = "0.23.1"
4+
version = "0.24.0"

‎streaming_data_types/area_detector_ad00.py‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
from struct import pack
2+
from typing import List, NamedTuple, Union
3+
4+
import flatbuffers
5+
import numpy as np
6+
7+
import streaming_data_types.fbschemas.area_detector_ad00.Attribute as ADArAttribute
8+
from streaming_data_types.fbschemas.area_detector_ad00 import ad00_ADArray
9+
from streaming_data_types.fbschemas.area_detector_ad00.DType import DType
10+
from streaming_data_types.utils import check_schema_identifier
11+
12+
FILE_IDENTIFIER = b"ad00"
13+
14+
15+
class Attribute:
16+
def __init__(
17+
self,
18+
name: str,
19+
description: str,
20+
source: str,
21+
data: Union[np.ndarray, str, int, float],
22+
):
23+
self.name = name
24+
self.description = description
25+
self.source = source
26+
self.data = data
27+
28+
def __eq__(self, other):
29+
data_is_equal = type(self.data) == type(other.data) # noqa: E721
30+
if type(self.data) is np.ndarray:
31+
data_is_equal = data_is_equal and np.array_equal(self.data, other.data)
32+
else:
33+
data_is_equal = data_is_equal and self.data == other.data
34+
return (
35+
self.name == other.name
36+
and self.description == other.description
37+
and self.source == other.source
38+
and data_is_equal
39+
)
40+
41+
42+
def serialise_ad00(
43+
source_name: str,
44+
unique_id: int,
45+
timestamp_ns: int,
46+
data: Union[np.ndarray, str],
47+
attributes: List[Attribute] = [],
48+
) -> bytes:
49+
builder = flatbuffers.Builder(1024)
50+
builder.ForceDefaults(True)
51+
52+
type_map = {
53+
np.dtype("uint8"): DType.uint8,
54+
np.dtype("int8"): DType.int8,
55+
np.dtype("uint16"): DType.uint16,
56+
np.dtype("int16"): DType.int16,
57+
np.dtype("uint32"): DType.uint32,
58+
np.dtype("int32"): DType.int32,
59+
np.dtype("uint64"): DType.uint64,
60+
np.dtype("int64"): DType.int64,
61+
np.dtype("float32"): DType.float32,
62+
np.dtype("float64"): DType.float64,
63+
}
64+
65+
if type(data) is str:
66+
data = np.frombuffer(data.encode(), np.uint8)
67+
data_type = DType.c_string
68+
else:
69+
data_type = type_map[data.dtype]
70+
71+
# Build dims
72+
dims_offset = builder.CreateNumpyVector(np.asarray(data.shape))
73+
74+
# Build data
75+
data_offset = builder.CreateNumpyVector(data.flatten().view(np.uint8))
76+
77+
source_name_offset = builder.CreateString(source_name)
78+
79+
temp_attributes = []
80+
for item in attributes:
81+
if type(item.data) is np.ndarray:
82+
attr_data_type = type_map[item.data.dtype]
83+
attr_data = item.data
84+
elif type(item.data) is str:
85+
attr_data_type = DType.c_string
86+
attr_data = np.frombuffer(item.data.encode(), np.uint8)
87+
elif type(item.data) is int:
88+
attr_data_type = DType.int64
89+
attr_data = np.frombuffer(pack("q", item.data), np.uint8)
90+
elif type(item.data) is float:
91+
attr_data_type = DType.float64
92+
attr_data = np.frombuffer(pack("d", item.data), np.uint8)
93+
attr_name_offset = builder.CreateString(item.name)
94+
attr_desc_offset = builder.CreateString(item.description)
95+
attr_src_offset = builder.CreateString(item.source)
96+
attr_data_offset = builder.CreateNumpyVector(attr_data.flatten().view(np.uint8))
97+
ADArAttribute.AttributeStart(builder)
98+
ADArAttribute.AttributeAddName(builder, attr_name_offset)
99+
ADArAttribute.AttributeAddDescription(builder, attr_desc_offset)
100+
ADArAttribute.AttributeAddSource(builder, attr_src_offset)
101+
ADArAttribute.AttributeAddDataType(builder, attr_data_type)
102+
ADArAttribute.AttributeAddData(builder, attr_data_offset)
103+
attr_offset = ADArAttribute.AttributeEnd(builder)
104+
temp_attributes.append(attr_offset)
105+
106+
ad00_ADArray.ad00_ADArrayStartAttributesVector(builder, len(attributes))
107+
for item in reversed(temp_attributes):
108+
builder.PrependUOffsetTRelative(item)
109+
attributes_offset = builder.EndVector()
110+
111+
# Build the actual buffer
112+
ad00_ADArray.ad00_ADArrayStart(builder)
113+
ad00_ADArray.ad00_ADArrayAddSourceName(builder, source_name_offset)
114+
ad00_ADArray.ad00_ADArrayAddDataType(builder, data_type)
115+
ad00_ADArray.ad00_ADArrayAddDimensions(builder, dims_offset)
116+
ad00_ADArray.ad00_ADArrayAddId(builder, unique_id)
117+
ad00_ADArray.ad00_ADArrayAddData(builder, data_offset)
118+
ad00_ADArray.ad00_ADArrayAddTimestamp(builder, timestamp_ns)
119+
ad00_ADArray.ad00_ADArrayAddAttributes(builder, attributes_offset)
120+
array_message = ad00_ADArray.ad00_ADArrayEnd(builder)
121+
122+
builder.Finish(array_message, file_identifier=FILE_IDENTIFIER)
123+
return bytes(builder.Output())
124+
125+
126+
ADArray = NamedTuple(
127+
"ADArray",
128+
(
129+
("source_name", str),
130+
("unique_id", int),
131+
("timestamp_ns", int),
132+
("dimensions", np.ndarray),
133+
("data", np.ndarray),
134+
("attributes", List[Attribute]),
135+
),
136+
)
137+
138+
139+
def get_payload_data(fb_arr) -> np.ndarray:
140+
return get_data(fb_arr).reshape(fb_arr.DimensionsAsNumpy())
141+
142+
143+
def get_data(fb_arr) -> np.ndarray:
144+
"""
145+
Converts the data array into the correct type.
146+
"""
147+
raw_data = fb_arr.DataAsNumpy()
148+
type_map = {
149+
DType.uint8: np.uint8,
150+
DType.int8: np.int8,
151+
DType.uint16: np.uint16,
152+
DType.int16: np.int16,
153+
DType.uint32: np.uint32,
154+
DType.int32: np.int32,
155+
DType.uint64: np.uint64,
156+
DType.int64: np.int64,
157+
DType.float32: np.float32,
158+
DType.float64: np.float64,
159+
}
160+
return raw_data.view(type_map[fb_arr.DataType()])
161+
162+
163+
def deserialise_ad00(buffer: Union[bytearray, bytes]) -> ADArray:
164+
check_schema_identifier(buffer, FILE_IDENTIFIER)
165+
166+
ad_array = ad00_ADArray.ad00_ADArray.GetRootAsad00_ADArray(buffer, 0)
167+
unique_id = ad_array.Id()
168+
if ad_array.DataType() == DType.c_string:
169+
data = ad_array.DataAsNumpy().tobytes().decode()
170+
else:
171+
data = get_payload_data(ad_array)
172+
173+
attributes_list = []
174+
for i in range(ad_array.AttributesLength()):
175+
attribute_ptr = ad_array.Attributes(i)
176+
if attribute_ptr.DataType() == DType.c_string:
177+
attr_data = attribute_ptr.DataAsNumpy().tobytes().decode()
178+
else:
179+
attr_data = get_data(attribute_ptr)
180+
temp_attribute = Attribute(
181+
name=attribute_ptr.Name().decode(),
182+
description=attribute_ptr.Description().decode(),
183+
source=attribute_ptr.Source().decode(),
184+
data=attr_data,
185+
)
186+
if type(temp_attribute.data) is np.ndarray and len(temp_attribute.data) == 1:
187+
if np.issubdtype(temp_attribute.data.dtype, np.floating):
188+
temp_attribute.data = float(temp_attribute.data[0])
189+
elif np.issubdtype(temp_attribute.data.dtype, np.integer):
190+
temp_attribute.data = int(temp_attribute.data[0])
191+
attributes_list.append(temp_attribute)
192+
193+
return ADArray(
194+
source_name=ad_array.SourceName().decode(),
195+
unique_id=unique_id,
196+
timestamp_ns=ad_array.Timestamp(),
197+
dimensions=tuple(ad_array.DimensionsAsNumpy()),
198+
data=data,
199+
attributes=attributes_list,
200+
)

‎streaming_data_types/fbschemas/area_detector_ad00/Attribute.py‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
# automatically generated by the FlatBuffers compiler, do not modify
2+
3+
# namespace:
4+
5+
import flatbuffers
6+
from flatbuffers.compat import import_numpy
7+
8+
np = import_numpy()
9+
10+
11+
class Attribute(object):
12+
__slots__ = ["_tab"]
13+
14+
@classmethod
15+
def GetRootAs(cls, buf, offset=0):
16+
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
17+
x = Attribute()
18+
x.Init(buf, n + offset)
19+
return x
20+
21+
@classmethod
22+
def GetRootAsAttribute(cls, buf, offset=0):
23+
"""This method is deprecated. Please switch to GetRootAs."""
24+
return cls.GetRootAs(buf, offset)
25+
26+
@classmethod
27+
def AttributeBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
28+
return flatbuffers.util.BufferHasIdentifier(
29+
buf, offset, b"\x61\x64\x30\x30", size_prefixed=size_prefixed
30+
)
31+
32+
# Attribute
33+
def Init(self, buf, pos):
34+
self._tab = flatbuffers.table.Table(buf, pos)
35+
36+
# Attribute
37+
def Name(self):
38+
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
39+
if o != 0:
40+
return self._tab.String(o + self._tab.Pos)
41+
return None
42+
43+
# Attribute
44+
def Description(self):
45+
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
46+
if o != 0:
47+
return self._tab.String(o + self._tab.Pos)
48+
return None
49+
50+
# Attribute
51+
def Source(self):
52+
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
53+
if o != 0:
54+
return self._tab.String(o + self._tab.Pos)
55+
return None
56+
57+
# Attribute
58+
def DataType(self):
59+
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
60+
if o != 0:
61+
return self._tab.Get(flatbuffers.number_types.Int8Flags, o + self._tab.Pos)
62+
return 0
63+
64+
# Attribute
65+
def Data(self, j):
66+
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
67+
if o != 0:
68+
a = self._tab.Vector(o)
69+
return self._tab.Get(
70+
flatbuffers.number_types.Uint8Flags,
71+
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1),
72+
)
73+
return 0
74+
75+
# Attribute
76+
def DataAsNumpy(self):
77+
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
78+
if o != 0:
79+
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
80+
return 0
81+
82+
# Attribute
83+
def DataLength(self):
84+
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
85+
if o != 0:
86+
return self._tab.VectorLen(o)
87+
return 0
88+
89+
# Attribute
90+
def DataIsNone(self):
91+
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
92+
return o == 0
93+
94+
95+
def AttributeStart(builder):
96+
builder.StartObject(5)
97+
98+
99+
def Start(builder):
100+
AttributeStart(builder)
101+
102+
103+
def AttributeAddName(builder, name):
104+
builder.PrependUOffsetTRelativeSlot(
105+
0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0
106+
)
107+
108+
109+
def AddName(builder: flatbuffers.Builder, name: int):
110+
AttributeAddName(builder, name)
111+
112+
113+
def AttributeAddDescription(builder, description):
114+
builder.PrependUOffsetTRelativeSlot(
115+
1, flatbuffers.number_types.UOffsetTFlags.py_type(description), 0
116+
)
117+
118+
119+
def AddDescription(builder: flatbuffers.Builder, description: int):
120+
AttributeAddDescription(builder, description)
121+
122+
123+
def AttributeAddSource(builder, source):
124+
builder.PrependUOffsetTRelativeSlot(
125+
2, flatbuffers.number_types.UOffsetTFlags.py_type(source), 0
126+
)
127+
128+
129+
def AddSource(builder: flatbuffers.Builder, source: int):
130+
AttributeAddSource(builder, source)
131+
132+
133+
def AttributeAddDataType(builder, dataType):
134+
builder.PrependInt8Slot(3, dataType, 0)
135+
136+
137+
def AddDataType(builder: flatbuffers.Builder, dataType: int):
138+
AttributeAddDataType(builder, dataType)
139+
140+
141+
def AttributeAddData(builder, data):
142+
builder.PrependUOffsetTRelativeSlot(
143+
4, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0
144+
)
145+
146+
147+
def AddData(builder: flatbuffers.Builder, data: int):
148+
AttributeAddData(builder, data)
149+
150+
151+
def AttributeStartDataVector(builder, numElems):
152+
return builder.StartVector(1, numElems, 1)
153+
154+
155+
def StartDataVector(builder, numElems: int) -> int:
156+
return AttributeStartDataVector(builder, numElems)
157+
158+
159+
def AttributeEnd(builder):
160+
return builder.EndObject()
161+
162+
163+
def End(builder):
164+
return AttributeEnd(builder)

0 commit comments

Comments
 (0)