Removing obsolete problems, merging.

PiperOrigin-RevId: 162020990

Author: Lukasz Kaiser

tensor2tensor/bin/t2t-datagen

@@ -101,6 +101,14 @@ _SUPPORTED_PROBLEM_GENERATORS = {
     "algorithmic_algebra_inverse": (
         lambda: algorithmic_math.algebra_inverse(26, 0, 2, 100000),
         lambda: algorithmic_math.algebra_inverse(26, 3, 3, 10000)),
+    "ice_parsing_tokens": (
+        lambda: wmt.tabbed_parsing_token_generator(FLAGS.tmp_dir,
+                                                   True, "ice", 2**13, 2**8),
+        lambda: wmt.tabbed_parsing_token_generator(FLAGS.tmp_dir,
+                                                   False, "ice", 2**13, 2**8)),
+    "ice_parsing_characters": (
+        lambda: wmt.tabbed_parsing_character_generator(FLAGS.tmp_dir, True),
+        lambda: wmt.tabbed_parsing_character_generator(FLAGS.tmp_dir, False)),
     "wmt_parsing_tokens_8k": (
         lambda: wmt.parsing_token_generator(FLAGS.tmp_dir, True, 2**13),
         lambda: wmt.parsing_token_generator(FLAGS.tmp_dir, False, 2**13)),
@@ -109,11 +117,6 @@ _SUPPORTED_PROBLEM_GENERATORS = {
                                                     2**14, 2**9),
         lambda: wsj_parsing.parsing_token_generator(FLAGS.tmp_dir, False,
                                                     2**14, 2**9)),
-    "wsj_parsing_tokens_32k": (
-        lambda: wsj_parsing.parsing_token_generator(FLAGS.tmp_dir, True,
-                                                    2**15, 2**9),
-        lambda: wsj_parsing.parsing_token_generator(FLAGS.tmp_dir, False,
-                                                    2**15, 2**9)),
     "wmt_enfr_characters": (
         lambda: wmt.enfr_character_generator(FLAGS.tmp_dir, True),
         lambda: wmt.enfr_character_generator(FLAGS.tmp_dir, False)),
@@ -139,6 +142,12 @@ _SUPPORTED_PROBLEM_GENERATORS = {
         lambda: wmt.ende_wordpiece_token_generator(FLAGS.tmp_dir, True, 2**15),
         lambda: wmt.ende_wordpiece_token_generator(FLAGS.tmp_dir, False, 2**15)
     ),
+    "wmt_zhen_tokens_32k": (
+        lambda: wmt.zhen_wordpiece_token_generator(FLAGS.tmp_dir, True,
+                                                   2**15, 2**15),
+        lambda: wmt.zhen_wordpiece_token_generator(FLAGS.tmp_dir, False,
+                                                   2**15, 2**15)
+    ),
     "lm1b_32k": (
         lambda: lm1b.generator(FLAGS.tmp_dir, True),
         lambda: lm1b.generator(FLAGS.tmp_dir, False)
@@ -159,26 +168,9 @@ _SUPPORTED_PROBLEM_GENERATORS = {
     "image_cifar10_test": (
         lambda: image.cifar10_generator(FLAGS.tmp_dir, True, 50000),
         lambda: image.cifar10_generator(FLAGS.tmp_dir, False, 10000)),
-    "image_mscoco_characters_tune": (
-        lambda: image.mscoco_generator(FLAGS.tmp_dir, True, 70000),
-        lambda: image.mscoco_generator(FLAGS.tmp_dir, True, 10000, 70000)),
     "image_mscoco_characters_test": (
         lambda: image.mscoco_generator(FLAGS.tmp_dir, True, 80000),
         lambda: image.mscoco_generator(FLAGS.tmp_dir, False, 40000)),
-    "image_mscoco_tokens_8k_tune": (
-        lambda: image.mscoco_generator(
-            FLAGS.tmp_dir,
-            True,
-            70000,
-            vocab_filename="tokens.vocab.%d" % 2**13,
-            vocab_size=2**13),
-        lambda: image.mscoco_generator(
-            FLAGS.tmp_dir,
-            True,
-            10000,
-            70000,
-            vocab_filename="tokens.vocab.%d" % 2**13,
-            vocab_size=2**13)),
     "image_mscoco_tokens_8k_test": (
         lambda: image.mscoco_generator(
             FLAGS.tmp_dir,
@@ -192,20 +184,6 @@ _SUPPORTED_PROBLEM_GENERATORS = {
             40000,
             vocab_filename="tokens.vocab.%d" % 2**13,
             vocab_size=2**13)),
-    "image_mscoco_tokens_32k_tune": (
-        lambda: image.mscoco_generator(
-            FLAGS.tmp_dir,
-            True,
-            70000,
-            vocab_filename="tokens.vocab.%d" % 2**15,
-            vocab_size=2**15),
-        lambda: image.mscoco_generator(
-            FLAGS.tmp_dir,
-            True,
-            10000,
-            70000,
-            vocab_filename="tokens.vocab.%d" % 2**15,
-            vocab_size=2**15)),
     "image_mscoco_tokens_32k_test": (
         lambda: image.mscoco_generator(
             FLAGS.tmp_dir,
@@ -386,7 +364,7 @@ def generate_data_for_problem(problem):
 
 def generate_data_for_registered_problem(problem_name):
   problem = registry.problem(problem_name)
-  problem.generate_data(FLAGS.data_dir)
+  problem.generate_data(FLAGS.data_dir, FLAGS.tmp_dir)
 
 
 if __name__ == "__main__":

tensor2tensor/data_generators/algorithmic.py

@@ -36,7 +36,7 @@ class AlgorithmicIdentityBinary40(problem.Problem):
   def num_symbols(self):
     return 2
 
-  def generate_data(self, data_dir):
+  def generate_data(self, data_dir, _):
     utils.generate_files(
         identity_generator(self.num_symbols, 40, 100000),
         self.training_filepaths(data_dir, 100))

tensor2tensor/data_generators/generator_utils.py

@@ -300,6 +300,46 @@ def get_or_generate_vocab(tmp_dir, vocab_filename, vocab_size, sources=None):
   return vocab
 
 
+def get_or_generate_tabbed_vocab(tmp_dir, source_filename,
+                                 index, vocab_filename, vocab_size):
+  r"""Generate a vocabulary from a tabbed source file.
+
+  The source is a file of source, target pairs, where each line contains
+  a source string and a target string, separated by a tab ('\t') character.
+  The index parameter specifies 0 for the source or 1 for the target.
+
+  Args:
+    tmp_dir: path to the temporary directory.
+    source_filename: the name of the tab-separated source file.
+    index: index.
+    vocab_filename: the name of the vocabulary file.
+    vocab_size: vocabulary size.
+
+  Returns:
+    The vocabulary.
+  """
+  vocab_filepath = os.path.join(tmp_dir, vocab_filename)
+  if os.path.exists(vocab_filepath):
+    vocab = text_encoder.SubwordTextEncoder(vocab_filepath)
+    return vocab
+
+  # Use Tokenizer to count the word occurrences.
+  filepath = os.path.join(tmp_dir, source_filename)
+  with tf.gfile.GFile(filepath, mode="r") as source_file:
+    for line in source_file:
+      line = line.strip()
+      if line and "\t" in line:
+        parts = line.split("\t", maxsplit=1)
+        part = parts[index].strip()
+        _ = tokenizer.encode(text_encoder.native_to_unicode(part))
+
+  vocab = text_encoder.SubwordTextEncoder.build_to_target_size(
+      vocab_size, tokenizer.token_counts, 1,
+      min(1e3, vocab_size + text_encoder.NUM_RESERVED_TOKENS))
+  vocab.store_to_file(vocab_filepath)
+  return vocab
+
+
 def read_records(filename):
   reader = tf.python_io.tf_record_iterator(filename)
   records = []

tensor2tensor/data_generators/image.py

@@ -33,6 +33,9 @@
 from six.moves import xrange  # pylint: disable=redefined-builtin
 from six.moves import zip  # pylint: disable=redefined-builtin
 from tensor2tensor.data_generators import generator_utils
+from tensor2tensor.data_generators import problem
+from tensor2tensor.data_generators import text_encoder
+from tensor2tensor.utils import registry
 
 import tensorflow as tf
 
@@ -300,3 +303,47 @@ def mscoco_generator(tmp_dir,
             "image/height": [height],
             "image/width": [width]
         }
+
+# French street names dataset.
+
+
+@registry.register_problem
+class ImageFSNS(problem.Problem):
+  """Problem spec for French Street Name recognition."""
+
+  def generate_data(self, data_dir, tmp_dir):
+    list_url = ("https://raw.githubusercontent.com/tensorflow/models/master/"
+                "street/python/fsns_urls.txt")
+    fsns_urls = generator_utils.maybe_download(
+        tmp_dir, "fsns_urls.txt", list_url)
+    fsns_files = [f.strip() for f in open(fsns_urls, "r")
+                  if f.startswith("http://")]
+    for url in fsns_files:
+      if "/train/train" in url:
+        generator_utils.maybe_download(
+            data_dir, "image_fsns-train" + url[-len("-00100-of-00512"):], url)
+      elif "/validation/validation" in url:
+        generator_utils.maybe_download(
+            data_dir, "image_fsns-dev" + url[-len("-00100-of-00512"):], url)
+      elif "charset" in url:
+        generator_utils.maybe_download(
+            data_dir, "charset_size134.txt", url)
+
+  def hparams(self, defaults, model_hparams):
+    p = defaults
+    p.input_modality = {"inputs": (registry.Modalities.IMAGE, None)}
+    # This vocab file must be present within the data directory.
+    vocab_filename = os.path.join(model_hparams.data_dir, "charset_size134.txt")
+    subtokenizer = text_encoder.SubwordTextEncoder(vocab_filename)
+    p.target_modality = (registry.Modalities.SYMBOL, subtokenizer.vocab_size)
+    p.vocabulary = {
+        "inputs": text_encoder.TextEncoder(),
+        "targets": subtokenizer,
+    }
+    p.batch_size_multiplier = 256
+    p.max_expected_batch_size_per_shard = 2
+    vocab_size = 144
+    p.input_modality = {"inputs": (registry.Modalities.SYMBOL, vocab_size)}
+    p.target_modality = (registry.Modalities.SYMBOL, vocab_size)
+    p.input_space_id = problem.SpaceID.DIGIT_0
+    p.target_space_id = problem.SpaceID.DIGIT_1

tensor2tensor/data_generators/problem.py

@@ -59,6 +59,14 @@ class SpaceID(object):
   PARSE_CHR = 14
   # Parse tokens
   PARSE_TOK = 15
+  # Chinese tokens
+  ZH_TOK = 16
+  # Icelandic characters
+  ICE_CHAR = 17
+  # Icelandic tokens
+  ICE_TOK = 18
+  # Icelandic parse tokens
+  ICE_PARSE_TOK = 19
 
 
 class Problem(object):
@@ -97,7 +105,7 @@ class Problem(object):
   # BEGIN SUBCLASS INTERFACE
   # ============================================================================
 
-  def generate_data(self, data_dir):
+  def generate_data(self, data_dir, tmp_dir):
     raise NotImplementedError()
 
   def hparams(self, defaults, model_hparams):