IndexLookup, Accumulator.

Author: Oceania2018

src/TensorFlowNET.Core/APIs/tf.strings.cs

@@ -64,6 +64,9 @@ public Tensor substr(Tensor input, int pos, int len,
             public Tensor substr(string input, int pos, int len,
                     string name = null, string @uint = "BYTE")
                 => ops.substr(input, pos, len, @uint: @uint, name: name);
+
+            public Tensor split(Tensor input, string sep = "", int maxsplit = -1, string name = null)
+                => ops.string_split_v2(input, sep: sep, maxsplit : maxsplit, name : name);
         }
     }
 }

src/TensorFlowNET.Core/Data/DatasetV2.cs

@@ -68,6 +68,17 @@ public IDatasetV2 map(Func<Tensors, Tensors> map_func,
         public IDatasetV2 map(Func<Tensors, Tensors> map_func, int num_parallel_calls)
             => new ParallelMapDataset(this, map_func, num_parallel_calls: num_parallel_calls);
 
+        public OwnedIterator make_one_shot_iterator()
+        {
+            if (tf.Context.executing_eagerly())
+            {
+                // with ops.colocate_with(self._variant_tensor)
+                return new OwnedIterator(this);
+            }
+
+            throw new NotImplementedException("");
+        }
+
         public IDatasetV2 flat_map(Func<Tensor, IDatasetV2> map_func)
             => new FlatMapDataset(this, map_func);
 

src/TensorFlowNET.Core/Data/IDatasetV2.cs

@@ -72,6 +72,8 @@ IDatasetV2 map(Func<Tensors, Tensors> map_func,
         IDatasetV2 map(Func<Tensors, Tensors> map_func,
             int num_parallel_calls);
 
+        OwnedIterator make_one_shot_iterator();
+
         IDatasetV2 flat_map(Func<Tensor, IDatasetV2> map_func);
 
         IDatasetV2 model(AutotuneAlgorithm algorithm, long cpu_budget);

src/TensorFlowNET.Core/Data/OwnedIterator.cs

@@ -26,6 +26,7 @@ void _create_iterator(IDatasetV2 dataset)
             dataset = dataset.apply_options();
             _dataset = dataset;
             _element_spec = dataset.element_spec;
+            // _flat_output_types = 
             (_iterator_resource, _deleter) = ops.anonymous_iterator_v2(_dataset.output_types, _dataset.output_shapes);
             ops.make_iterator(dataset.variant_tensor, _iterator_resource);
         }

src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/TextVectorizationArgs.cs

@@ -11,5 +11,6 @@ public class TextVectorizationArgs : PreprocessingLayerArgs
         public int MaxTokens { get; set; } = -1;
         public string OutputMode { get; set; } = "int";
         public int OutputSequenceLength { get; set; } = -1;
+        public string[] Vocabulary { get; set; }
     }
 }

src/TensorFlowNET.Core/Operations/string_ops.cs

@@ -41,5 +41,10 @@ public Tensor substr<T>(T input, int pos, int len,
                 string @uint = "BYTE", string name = null)
             => tf.Context.ExecuteOp("Substr", name, new ExecuteOpArgs(input, pos, len)
                 .SetAttributes(new { unit = @uint }));
+
+        public Tensor string_split_v2(Tensor input, string sep = "", int maxsplit = -1, string name = null)
+        {
+            return null;
+        }
     }
 }

src/TensorFlowNET.Keras/Engine/CombinerPreprocessingLayer.cs

@@ -8,11 +8,23 @@ namespace Tensorflow.Keras.Engine
     public class CombinerPreprocessingLayer : Layer
     {
         PreprocessingLayerArgs args;
+        protected ICombiner combiner;
+        protected bool _previously_updated;
 
         public CombinerPreprocessingLayer(PreprocessingLayerArgs args)
             : base(args)
         {
-            
+            _previously_updated = false;
+        }
+
+        public virtual void adapt(IDatasetV2 data, bool reset_state = true)
+        {
+            IAccumulator accumulator;
+            if (!reset_state)
+                accumulator = combiner.Restore();
+
+            var next_data = data.make_one_shot_iterator();
+            var data_element = next_data.next();
         }
     }
 }

src/TensorFlowNET.Keras/Engine/Interfaces/IAccumulator.cs

@@ -0,0 +1,10 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.Engine
+{
+    public interface IAccumulator
+    {
+    }
+}

src/TensorFlowNET.Keras/Engine/Interfaces/ICombiner.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.Engine
+{
+    /// <summary>
+    /// Functional object that defines a shardable computation.
+    /// </summary>
+    public interface ICombiner
+    {
+        void Compute(Tensor values, IAccumulator accumulator = null);
+        void Merge();
+        void Extract();
+        IAccumulator Restore();
+        void Serialize();
+        void Deserialize();
+    }
+}

src/TensorFlowNET.Keras/Layers/Preprocessing/IndexLookup.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.ArgsDefinition;
+using Tensorflow.Keras.Engine;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class IndexLookup : CombinerPreprocessingLayer
+    {
+        public IndexLookup(int max_tokens = -1,
+            int num_oov_indices = 1,
+            string mask_token = "",
+            string oov_token = "[UNK]",
+            string encoding = "utf-8",
+            bool invert = false) : base(new PreprocessingLayerArgs())
+        {
+            var num_mask_tokens = mask_token == null ? 0 : 1;
+            var vocab_size = max_tokens - (num_oov_indices + num_mask_tokens);
+            combiner = new IndexLookupCombiner(vocab_size, mask_token);
+        }
+
+        public override void adapt(IDatasetV2 data, bool reset_state = true)
+        {
+            if (!reset_state)
+                throw new ValueError("IndexLookup does not support streaming adapts.");
+            base.adapt(data, reset_state);
+        }
+    }
+}