Iceparser adaptations

Author: vthorsteinsson

tensor2tensor/data_generators/generator_utils.py

@@ -300,7 +300,8 @@ def get_or_generate_tabbed_vocab(tmp_dir, source_filename, index, vocab_filename
         _ = tokenizer.encode(text_encoder.native_to_unicode(part))
 
   vocab = text_encoder.SubwordTextEncoder.build_to_target_size(
-      vocab_size, tokenizer.token_counts, 1, 1e3)
+      vocab_size, tokenizer.token_counts, 1,
+      min(1e3, vocab_size + text_encoder.NUM_RESERVED_TOKENS))
   vocab.store_to_file(vocab_filepath)
   return vocab
 

tensor2tensor/data_generators/problem_hparams.py

@@ -178,6 +178,7 @@ def default_problem_hparams():
       #   15: Parse tokens
       #   16: Icelandic characters
       #   17: Icelandic tokens
+      #   18: Icelandic parse tokens
       # Add more above if needed.
       input_space_id=0,
       target_space_id=0,
@@ -550,20 +551,6 @@ def wmt_parsing_tokens(model_hparams, wrong_vocab_size):
   return p
 
 
-def wmt_tabbed_parsing_characters(model_hparams):
-  p = default_problem_hparams()
-  p.input_modality = {"inputs": (registry.Modalities.SYMBOL, 256)}
-  p.target_modality = (registry.Modalities.SYMBOL, 256)
-  p.vocabulary = {
-      "inputs": text_encoder.ByteTextEncoder(),
-      "targets": text_encoder.ByteTextEncoder(),
-  }
-  p.loss_multiplier = 2.0
-  p.input_space_id = 2
-  p.target_space_id = 14
-  return p
-
-
 def wsj_parsing_tokens(model_hparams, prefix,
                        wrong_source_vocab_size,
                        wrong_target_vocab_size):
@@ -604,6 +591,37 @@ def wsj_parsing_tokens(model_hparams, prefix,
   return p
 
 
+def ice_parsing_tokens(model_hparams, wrong_source_vocab_size):
+  """Icelandic to parse tree translation benchmark.
+
+  Args:
+    model_hparams: a tf.contrib.training.HParams
+  Returns:
+    a tf.contrib.training.HParams
+  """
+  p = default_problem_hparams()
+  # This vocab file must be present within the data directory.
+  source_vocab_filename = os.path.join(
+      model_hparams.data_dir,
+      "ice_source.tokens.vocab.%d" % wrong_source_vocab_size)
+  target_vocab_filename = os.path.join(
+      model_hparams.data_dir,
+      "ice_target.tokens.vocab.256")
+  source_subtokenizer = text_encoder.SubwordTextEncoder(source_vocab_filename)
+  target_subtokenizer = text_encoder.SubwordTextEncoder(target_vocab_filename)
+  p.input_modality = {
+      "inputs": (registry.Modalities.SYMBOL, source_subtokenizer.vocab_size)
+  }
+  p.target_modality = (registry.Modalities.SYMBOL, 256)
+  p.vocabulary = {
+      "inputs": source_subtokenizer,
+      "targets": target_subtokenizer,
+  }
+  p.input_space_id = 17 # Icelandic tokens
+  p.target_space_id = 18 # Icelandic parse tokens
+  return p
+
+
 def image_cifar10(unused_model_hparams):
   """CIFAR-10."""
   p = default_problem_hparams()
@@ -723,7 +741,7 @@ def img2img_imagenet(unused_model_hparams):
     "lmptb_10k": lmptb_10k,
     "wmt_parsing_characters": wmt_parsing_characters,
     "ice_parsing_characters": wmt_parsing_characters,
-    "ice_parsing_tokens": lambda p: wsj_parsing_tokens(p, "ice", 2**13, 2**8),
+    "ice_parsing_tokens": lambda p: ice_parsing_tokens(p, 2**13),
     "wmt_parsing_tokens_8k": lambda p: wmt_parsing_tokens(p, 2**13),
     "wsj_parsing_tokens_16k": lambda p: wsj_parsing_tokens(p, "wsj", 2**14, 2**9),
     "wsj_parsing_tokens_32k": lambda p: wsj_parsing_tokens(p, "wsj", 2**15, 2**9),

tensor2tensor/data_generators/text_encoder.py

@@ -44,6 +44,7 @@
 PAD = "<pad>"
 EOS = "<EOS>"
 RESERVED_TOKENS = [PAD, EOS]
+NUM_RESERVED_TOKENS = len(RESERVED_TOKENS)
 PAD_TOKEN = RESERVED_TOKENS.index(PAD) # Normally 0
 EOS_TOKEN = RESERVED_TOKENS.index(EOS) # Normally 1
 
@@ -55,7 +56,7 @@
 class TextEncoder(object):
   """Base class for converting from ints to/from human readable strings."""
 
-  def __init__(self, num_reserved_ids=2):
+  def __init__(self, num_reserved_ids=NUM_RESERVED_TOKENS):
     self._num_reserved_ids = num_reserved_ids
 
   def encode(self, s):
@@ -130,7 +131,7 @@ def vocab_size(self):
 class TokenTextEncoder(TextEncoder):
   """Encoder based on a user-supplied vocabulary."""
 
-  def __init__(self, vocab_filename, reverse=False, num_reserved_ids=2):
+  def __init__(self, vocab_filename, reverse=False, num_reserved_ids=NUM_RESERVED_TOKENS):
     """Initialize from a file, one token per line."""
     super(TokenTextEncoder, self).__init__(num_reserved_ids=num_reserved_ids)
     self._reverse = reverse
@@ -203,7 +204,7 @@ class SubwordTextEncoder(TextEncoder):
 
   """
 
-  def __init__(self, filename=None, num_reserved_ids=2):
+  def __init__(self, filename=None, num_reserved_ids=NUM_RESERVED_TOKENS):
     """Initialize and read from a file, if provided."""
     self._tokenizer = tokenizer.Tokenizer()
     if filename is not None:

tensor2tensor/models/transformer.py

@@ -356,12 +356,9 @@ def transformer_parsing_base():
 @registry.register_hparams
 def transformer_parsing_ice():
   """Hparams for parsing Icelandic text."""
-  hparams = transformer_parsing_base()
+  hparams = transformer_base_single_gpu()
   hparams.batch_size = 4096
-  hparams.batching_mantissa_bits = 2
-  hparams.hidden_size = 512
-  #hparams.max_length = 256
-  #hparams.hidden_size = 128
+  hparams.shared_embedding_and_softmax_weights = int(False)
   return hparams