Bug 1420225 - Read legacy files when scanning for Sources in transforms (#30)

Author: stasm

fluent/migrate/context.py

@@ -139,8 +139,12 @@ def maybe_add_localization(self, path):
         else:
             self.localization_resources[path] = collection
 
-    def add_transforms(self, path, reference, transforms):
-        """Define transforms for path using reference as template.
+    def add_transforms(self, target, reference, transforms):
+        """Define transforms for target using reference as template.
+
+        `target` is a path of the destination FTL file relative to the
+        localization directory. `reference` is a path to the template FTL
+        file relative to the reference directory.
 
         Each transform is an extended FTL node with `Transform` nodes as some
         values.  Transforms are stored in their lazy AST form until
@@ -169,34 +173,38 @@ def get_sources(acc, cur):
         else:
             # The reference file will be used by the merge function as
             # a template for serializing the merge results.
-            self.reference_resources[path] = ast
+            self.reference_resources[target] = ast
 
         for node in transforms:
             # Scan `node` for `Source` nodes and collect the information they
             # store into a set of dependencies.
             dependencies = fold(get_sources, node, set())
             # Set these sources as dependencies for the current transform.
-            self.dependencies[(path, node.id.name)] = dependencies
+            self.dependencies[(target, node.id.name)] = dependencies
+
+            # Read all legacy translation files defined in Source transforms.
+            for path in set(path for path, _ in dependencies):
+                self.maybe_add_localization(path)
 
-        path_transforms = self.transforms.setdefault(path, [])
+        path_transforms = self.transforms.setdefault(target, [])
         path_transforms += transforms
 
-        if path not in self.localization_resources:
-            fullpath = os.path.join(self.localization_dir, path)
+        if target not in self.localization_resources:
+            fullpath = os.path.join(self.localization_dir, target)
             try:
                 ast = self.read_ftl_resource(fullpath)
             except IOError:
                 logger = logging.getLogger('migrate')
                 logger.info(
                     'Localization file {} does not exist and '
-                    'it will be created'.format(path))
+                    'it will be created'.format(target))
             except UnicodeDecodeError:
                 logger = logging.getLogger('migrate')
                 logger.warn(
                     'Localization file {} will be re-created and some '
-                    'translations might be lost'.format(path))
+                    'translations might be lost'.format(target))
             else:
-                self.localization_resources[path] = ast
+                self.localization_resources[target] = ast
 
     def get_source(self, path, key):
         """Get an entity value from a localized legacy source.

tests/migrate/test_context.py

@@ -5,6 +5,11 @@
 import logging
 import unittest
 
+try:
+    import compare_locales
+except ImportError:
+    compare_locales = None
+
 import fluent.syntax.ast as FTL
 
 from fluent.migrate.errors import NotSupportedError, UnreadableReferenceError
@@ -18,6 +23,7 @@ def here(*parts):
     return os.path.join(dirname, *parts)
 
 
+@unittest.skipUnless(compare_locales, 'compare-locales requried')
 class TestMergeContext(unittest.TestCase):
     def setUp(self):
         self.ctx = MergeContext(
@@ -26,12 +32,6 @@ def setUp(self):
             localization_dir=here('fixtures/pl')
         )
 
-        try:
-            self.ctx.maybe_add_localization('aboutDownloads.dtd')
-            self.ctx.maybe_add_localization('aboutDownloads.properties')
-        except RuntimeError:
-            self.skipTest('compare-locales required')
-
     def test_hardcoded_node(self):
         self.ctx.add_transforms('aboutDownloads.ftl', 'aboutDownloads.ftl', [
             FTL.Message(
@@ -249,6 +249,7 @@ def test_missing_reference_file(self):
             self.ctx.add_transforms('some.ftl', 'missing.ftl', [])
 
 
+@unittest.skipUnless(compare_locales, 'compare-locales requried')
 class TestIncompleteLocalization(unittest.TestCase):
     def setUp(self):
         # Silence all logging.
@@ -260,11 +261,6 @@ def setUp(self):
             localization_dir=here('fixtures/pl')
         )
 
-        try:
-            self.ctx.maybe_add_localization('browser.dtd')
-        except RuntimeError:
-            self.skipTest('compare-locales required')
-
         self.ctx.add_transforms('toolbar.ftl', 'toolbar.ftl', [
             FTL.Message(
                 id=FTL.Identifier('urlbar-textbox'),
@@ -299,6 +295,7 @@ def test_missing_localization_file(self):
         )
 
 
+@unittest.skipUnless(compare_locales, 'compare-locales requried')
 class TestExistingTarget(unittest.TestCase):
     maxDiff = None
 
@@ -312,11 +309,6 @@ def setUp(self):
             localization_dir=here('fixtures/pl')
         )
 
-        try:
-            self.ctx.maybe_add_localization('privacy.dtd')
-        except RuntimeError:
-            self.skipTest('compare-locales required')
-
     def tearDown(self):
         # Resume logging.
         logging.disable(logging.NOTSET)

tests/migrate/test_context_real_examples.py

@@ -4,6 +4,11 @@
 import os
 import unittest
 
+try:
+    import compare_locales
+except ImportError:
+    compare_locales = None
+
 import fluent.syntax.ast as FTL
 
 from fluent.migrate.util import ftl_resource_to_json, to_json
@@ -19,6 +24,7 @@ def here(*parts):
     return os.path.join(dirname, *parts)
 
 
+@unittest.skipUnless(compare_locales, 'compare-locales requried')
 class TestMergeAboutDownloads(unittest.TestCase):
     def setUp(self):
         self.ctx = MergeContext(
@@ -27,12 +33,6 @@ def setUp(self):
             localization_dir=here('fixtures/pl')
         )
 
-        try:
-            self.ctx.maybe_add_localization('aboutDownloads.dtd')
-            self.ctx.maybe_add_localization('aboutDownloads.properties')
-        except RuntimeError:
-            self.skipTest('compare-locales required')
-
         self.ctx.add_transforms('aboutDownloads.ftl', 'aboutDownloads.ftl', [
             FTL.Message(
                 id=FTL.Identifier('title'),
@@ -279,6 +279,7 @@ def test_merge_context_some_messages(self):
         )
 
 
+@unittest.skipUnless(compare_locales, 'compare-locales requried')
 class TestMergeAboutDialog(unittest.TestCase):
     def setUp(self):
         self.ctx = MergeContext(
@@ -287,11 +288,6 @@ def setUp(self):
             localization_dir=here('fixtures/pl')
         )
 
-        try:
-            self.ctx.maybe_add_localization('aboutDialog.dtd')
-        except RuntimeError:
-            self.skipTest('compare-locales required')
-
         self.ctx.add_transforms('aboutDialog.ftl', 'aboutDialog.ftl', [
             FTL.Message(
                 id=FTL.Identifier('update-failed'),

tools/migrate/examples/about_dialog.py

@@ -9,8 +9,6 @@
 def migrate(ctx):
     """Migrate about:dialog, part {index}"""
 
-    ctx.maybe_add_localization('browser/chrome/browser/aboutDialog.dtd')
-
     ctx.add_transforms('browser/about_dialog.ftl', 'about_dialog.ftl', [
         FTL.Message(
             id=FTL.Identifier('update-failed'),

tools/migrate/examples/about_downloads.py

@@ -7,11 +7,6 @@
 def migrate(ctx):
     """Migrate about:download in Firefox for Android, part {index}"""
 
-    ctx.maybe_add_localization(
-        'mobile/android/chrome/aboutDownloads.dtd')
-    ctx.maybe_add_localization(
-        'mobile/android/chrome/aboutDownloads.properties')
-
     ctx.add_transforms('mobile/about_downloads.ftl', 'about_downloads.ftl', [
         FTL.Message(
             id=FTL.Identifier('title'),

tools/migrate/examples/bug_1291693.py

@@ -7,11 +7,6 @@
 def migrate(ctx):
     """Bug 1291693 - Migrate the menubar to FTL, part {index}"""
 
-    ctx.maybe_add_localization('browser/chrome/browser/browser.dtd')
-    ctx.maybe_add_localization('browser/chrome/browser/browser.properties')
-    ctx.maybe_add_localization('browser/branding/official/brand.dtd')
-    ctx.maybe_add_localization('browser/branding/official/brand.properties')
-
     ctx.add_transforms('browser/menubar.ftl', 'menubar.ftl', [
         FTL.Message(
             id=FTL.Identifier('file-menu'),