Skip to content

Commit f3cbd85

Browse files
Oceania2018Oceania2018
committed
np.frombuffer
1 parent 3c020dc commit f3cbd85

File tree

16 files changed

+86
-48
lines changed

16 files changed

+86
-48
lines changed

‎src/TensorFlowNET.Core/Data/MnistModelLoader.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ private NDArray ExtractImages(string file, int? limit = null)
123123

124124
bytestream.Read(buf, 0, buf.Length);
125125

126-
var data = np.frombuffer(buf, (rows, cols), np.@byte);
126+
var data = np.frombuffer(buf, new Shape(buf.Length), np.@byte);
127127
data = data.reshape((num_images, rows, cols, 1));
128128

129129
return data;

‎src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.Creation.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,21 @@ public NDArray eye(int N, int? M = null, int k = 0, TF_DataType dtype = TF_DataT
3636
return new NDArray(tensor);
3737
}
3838

39+
public NDArray frombuffer(byte[] bytes, string dtype)
40+
{
41+
if (dtype == ">u4")
42+
{
43+
var size = bytes.Length / sizeof(uint);
44+
var ints = new int[size];
45+
for (var index = 0; index < size; index++)
46+
ints[index] = bytes[0] * 256 + bytes[1] + bytes[2] * 256 + bytes[3];
47+
48+
return new NDArray(ints, shape: new Shape(size));
49+
}
50+
51+
throw new NotImplementedException("");
52+
}
53+
3954
public NDArray frombuffer(byte[] bytes, Shape shape, TF_DataType dtype)
4055
{
4156
return new NDArray(bytes, shape, dtype);

‎src/TensorFlowNET.Core/NumPy/NDArray.Index.cs‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ public NDArray this[NDArray mask]
4343
{
4444
get
4545
{
46+
if(mask.dtype == TF_DataType.TF_INT32)
47+
return GetData(mask.ToArray<int>());
48+
4649
throw new NotImplementedException("");
4750
}
4851

@@ -54,7 +57,39 @@ public NDArray this[NDArray mask]
5457

5558
NDArray GetData(IEnumerable<Slice> slices)
5659
{
57-
return _tensor[slices.ToArray()];
60+
var tensor = _tensor[slices.ToArray()];
61+
return new NDArray(tensor);
62+
}
63+
64+
NDArray GetData(int[] indices, int axis = 0)
65+
{
66+
if(axis == 0)
67+
{
68+
var dims = shape.as_int_list();
69+
dims[0] = indices.Length;
70+
71+
var array = np.ndarray(dims, dtype: dtype);
72+
73+
dims[0] = 1;
74+
var bytesize = new Shape(dims).size * dtype.get_datatype_size();
75+
76+
int dst_index = 0;
77+
foreach (var index in indices)
78+
{
79+
var src_offset = (ulong)ShapeHelper.GetOffset(shape, index);
80+
var dst_offset = (ulong)ShapeHelper.GetOffset(array.shape, dst_index++);
81+
unsafe
82+
{
83+
var src = (byte*)data + src_offset * dtypesize;
84+
var dst = (byte*)array.data.ToPointer() + dst_offset * dtypesize;
85+
System.Buffer.MemoryCopy(src, dst, bytesize, bytesize);
86+
}
87+
}
88+
89+
return array;
90+
}
91+
else
92+
throw new NotImplementedException("");
5893
}
5994

6095
void SetData(IEnumerable<Slice> slices, NDArray array)

‎src/TensorFlowNET.Core/Numpy/InfoOf.cs‎

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,14 @@ void Init(Tensor value, Shape? shape = null)
6565
{
6666
// created tensor in graph mode
6767
if (value.TensorDataPointer == IntPtr.Zero)
68-
value = tf.defaultSession.eval(value);
69-
70-
_tensor = new Tensor(value.TensorDataPointer, shape ?? value.shape, value.dtype);
68+
{
69+
if (!value.graph.building_function)
70+
{
71+
value = tf.defaultSession.eval(value);
72+
value = new Tensor(value.TensorDataPointer, shape ?? value.shape, value.dtype);
73+
}
74+
}
75+
_tensor = value;
7176
_tensor.SetReferencedByNDArray();
7277
}
7378

‎src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ public static NDArray full<T>(Shape shape, T fill_value)
3636
public static NDArray frombuffer(byte[] bytes, Shape shape, TF_DataType dtype)
3737
=> tf.numpy.frombuffer(bytes, shape, dtype);
3838

39+
public static NDArray frombuffer(byte[] bytes, string dtype)
40+
=> tf.numpy.frombuffer(bytes, dtype);
41+
3942
public static NDArray linspace<T>(T start, T stop, int num = 50, bool endpoint = true, bool retstep = false,
4043
TF_DataType dtype = TF_DataType.TF_DOUBLE, int axis = 0) where T : unmanaged
4144
=> tf.numpy.linspace(start, stop, num: num, endpoint: endpoint, retstep: retstep, dtype: dtype, axis: axis);

‎src/TensorFlowNET.Core/Numpy/Numpy.cs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,6 @@ public static bool array_equal(NDArray a, NDArray b)
7272
public static NDArray concatenate(NDArray[] arrays, int axis = 0)
7373
=> throw new NotImplementedException("");
7474

75-
public static NDArray frombuffer(byte[] bytes, string dtype)
76-
=> throw new NotImplementedException("");
77-
7875
public static bool allclose(NDArray a, NDArray b, double rtol = 1.0E-5, double atol = 1.0E-8,
7976
bool equal_nan = false) => throw new NotImplementedException("");
8077

‎src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ protected unsafe void InitTensor(Array array, Shape? shape = null)
129129
shape = shape ?? array.GetShape();
130130
var dtype = array.GetDataType();
131131

132-
if (shape.size == 0)
132+
if (shape.size == 0 && dtype != TF_DataType.TF_STRING)
133133
{
134134
_handle = TF_NewTensor(shape, dtype, null);
135135
return;

‎src/TensorFlowNET.Core/Tensors/Tensor.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,8 +215,11 @@ public TF_Output _as_tf_output()
215215

216216
public void SetReferencedByNDArray()
217217
{
218-
isReferencedByNDArray = true;
219-
_eagerTensorHandle = c_api.TFE_NewTensorHandle(_handle, tf.Status.Handle);
218+
if (_handle != IntPtr.Zero)
219+
{
220+
isReferencedByNDArray = true;
221+
_eagerTensorHandle = c_api.TFE_NewTensorHandle(_handle, tf.Status.Handle);
222+
}
220223
}
221224

222225
public Tensor MaybeMove()

‎src/TensorFlowNET.Core/Tensors/dtypes.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,12 +202,13 @@ public static int get_datatype_size(this TF_DataType type)
202202
{
203203
TF_DataType.TF_BOOL => sizeof(bool),
204204
TF_DataType.TF_UINT8 => sizeof(byte),
205+
TF_DataType.TF_INT8 => sizeof(byte),
205206
TF_DataType.TF_INT16 => sizeof(short),
206207
TF_DataType.TF_INT32 => sizeof(int),
207208
TF_DataType.TF_INT64 => sizeof(long),
208209
TF_DataType.TF_FLOAT => sizeof(float),
209210
TF_DataType.TF_DOUBLE => sizeof(double),
210-
_ => -1
211+
_ => throw new NotImplementedException("")
211212
};
212213

213214
public static Type as_numpy_dtype(this DataType type)

0 commit comments

Comments
 (0)