fix bug with bpe32k - prepend <pad> and <EOS> to vocab.

PiperOrigin-RevId: 177271941

Author: nshazeer

tensor2tensor/data_generators/translate_ende.py

@@ -99,8 +99,10 @@ def generator(self, data_dir, tmp_dir, train):
     token_tmp_path = os.path.join(tmp_dir, self.vocab_file)
     token_path = os.path.join(data_dir, self.vocab_file)
     tf.gfile.Copy(token_tmp_path, token_path, overwrite=True)
-    with tf.gfile.GFile(token_path, mode="a") as f:
-      f.write("UNK\n")  # Add UNK to the vocab.
+    with tf.gfile.GFile(token_path, mode="r") as f:
+      vocab_data = "<pad>\n<EOS>\n" + f.read() + "UNK\n"
+    with tf.gfile.GFile(token_path, mode="w") as f:
+      f.write(vocab_data)
     token_vocab = text_encoder.TokenTextEncoder(token_path, replace_oov="UNK")
     return translate.token_generator(train_path + ".en", train_path + ".de",
                                      token_vocab, EOS)