Keep activations in bidirectional LSTM (fixes: #2211) (#2234)

* Keep activations in bidirectional LSTM (fixes: #2211)
---------

Signed-off-by: Me <me@example.com>
Co-authored-by: Jay Zhang <36183870+fatcat-z@users.noreply.github.com>

Author: MaximeChurin

tests/test_lstm.py

@@ -775,6 +775,30 @@ def func(x):
             return tf.identity(y[0], name="output"), tf.identity(y[1], name="output1")
         self.run_test_case(func, {"input:0": x_val}, [], ["output:0", "output1:0"], rtol=1e-05, atol=1e-06)
 
+    @check_tf_min_version("2.0")
+    def test_keras_bilstm_recurrent_activation_is_hard_sigmoid(self):
+        in_shape = [10, 3]
+        x_val = np.random.uniform(size=[2, 10, 3]).astype(np.float32)
+
+        model_in = tf.keras.layers.Input(tuple(in_shape), batch_size=2)
+        x = tf.keras.layers.Bidirectional(
+            tf.keras.layers.LSTM(
+                units=5,
+                return_sequences=True,
+                return_state=True,
+                kernel_initializer=tf.random_uniform_initializer(0.0, 1.0, seed=42),
+                recurrent_initializer=tf.random_uniform_initializer(0.0, 1.0, seed=44),
+                bias_initializer=tf.random_uniform_initializer(0.0, 1.0, seed=43),
+                recurrent_activation="hard_sigmoid",
+            )
+        )(model_in)
+        model = tf.keras.models.Model(inputs=model_in, outputs=x)
+
+        def func(x):
+            y = model(x)
+            return tf.identity(y[0], name="output"), tf.identity(y[1], name="output1")
+        self.run_test_case(func, {"input:0": x_val}, [], ["output:0", "output1:0"], rtol=1e-05, atol=1e-06)
+
     @check_tf_min_version("2.0")
     @skip_tfjs("TFJS converts model incorrectly")
     def test_keras_lstm_sigmoid_dropout(self):

tf2onnx/rewriter/bilstm_rewriter.py

@@ -54,11 +54,19 @@ def process_bilstm(g, bi_lstms):
         if len(lstm_fw.inputs) > 4:
             lstm_inputs.extend([lstm_fw.input[4], h_node.output[0], c_node.output[0]])
 
-        attr = {"direction": "bidirectional"}
+        direction = "bidirectional"
+        attr = {"direction": direction}
         for name in rnn_utils.onnx_rnn_attr_mapping[rnn_utils.ONNX_RNN_TYPE.LSTM]:
             attr_val = lstm_fw.get_attr_value(name)
             if attr_val:
                 attr[name] = attr_val
+        # activation has to be took care, attr here is proto
+        activations = [act.decode("utf-8")
+                       for act in lstm_fw.get_attr_value("activations", [])]
+        activations += [act.decode("utf-8")
+                        for act in lstm_bw.get_attr_value("activations", [])]
+        if activations:
+            attr["activations"] = activations
 
         bi_lstm_node = g.make_node("LSTM", lstm_inputs, attr=attr, output_count=3)
         all_nodes.append(bi_lstm_node)