Use get_residual_fn to get the residual_fn in the transformer.

PiperOrigin-RevId: 163919630

Author: T2T Team

README.md

@@ -180,7 +180,7 @@ python -c "from tensor2tensor.models.transformer import Transformer"
 **Datasets** are all standardized on `TFRecord` files with `tensorflow.Example`
 protocol buffers. All datasets are registered and generated with the
 [data
-generator](https://github.com/tensorflow/tensor2tensor/tree/master/tensor2tensor/bin/t2t-datagen)
+generator](https://github.com/tensorflow/tensor2tensor/tree/master/tensor2tensor/data_generators/generator.py)
 and many common sequence datasets are already available for generation and use.
 
 ### Problems and Modalities

tensor2tensor/bin/t2t-datagen renamed to tensor2tensor/data_generators/generator.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #

tensor2tensor/models/transformer.py

@@ -56,12 +56,7 @@ def model_fn_body(self, features):
     (decoder_input, decoder_self_attention_bias) = transformer_prepare_decoder(
         targets, hparams)
 
-    def residual_fn(x, y):
-      return common_layers.residual_fn(x, y,
-                                       hparams.norm_type,
-                                       hparams.residual_dropout,
-                                       hparams.hidden_size,
-                                       epsilon=hparams.layer_norm_epsilon)
+    residual_fn = get_residual_fn(hparams)
 
     encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.residual_dropout)
     decoder_input = tf.nn.dropout(decoder_input, 1.0 - hparams.residual_dropout)
@@ -76,6 +71,17 @@ def residual_fn(x, y):
     return decoder_output
 
 
+def get_residual_fn(hparams):
+  """Get residual_fn."""
+  def residual_fn(x, y):
+    return common_layers.residual_fn(x, y,
+                                     hparams.norm_type,
+                                     hparams.residual_dropout,
+                                     hparams.hidden_size,
+                                     epsilon=hparams.layer_norm_epsilon)
+  return residual_fn
+
+
 def transformer_prepare_encoder(inputs, target_space, hparams):
   """Prepare one shard of the model for the encoder.
 

tensor2tensor/bin/t2t-trainer renamed to tensor2tensor/trainer.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # coding=utf-8
 # Copyright 2017 The Tensor2Tensor Authors.
 #