fix case when there is a repeated annotation inside variants. Added test.

Author: olveyra

scrapely/extraction/regionextract.py

@@ -27,7 +27,7 @@ def build_extraction_tree(template, type_descriptor, trace=True):
     extractors = BasicTypeExtractor.create(template.annotations, attribute_map)
     if trace:
         extractors = TraceExtractor.apply(template, extractors)
-    for cls in (AdjacentVariantExtractor, RepeatedDataExtractor, AdjacentVariantExtractor, RepeatedDataExtractor,
+    for cls in (RepeatedDataExtractor, AdjacentVariantExtractor, RepeatedDataExtractor, AdjacentVariantExtractor, RepeatedDataExtractor,
             RecordExtractor):
         extractors = cls.apply(template, extractors)
         if trace:
@@ -201,7 +201,7 @@ def __init__(self, prefix, suffix, extractors):
         self.annotation = copy.copy(self.extractor.annotation)
         self.annotation.end_index = extractors[-1].annotation.end_index
 
-    def extract(self, page, start_index, end_index, ignored_regions):
+    def extract(self, page, start_index, end_index, ignored_regions, **kwargs):
         """repeatedly find regions bounded by the repeated 
         prefix and suffix and extract them
         """
@@ -231,7 +231,7 @@ def extract(self, page, start_index, end_index, ignored_regions):
     def apply(template, extractors):
         tokens = template.page_tokens
         output_extractors = []
-        group_key = lambda x: x.extracted_item()
+        group_key = lambda x: (x.extracted_item(), x.annotation.variant_id)
         for extr_key, extraction_group in groupby(extractors, group_key):
             extraction_group = list(extraction_group)
             if extr_key is None or len(extraction_group) == 1:

scrapely/tests/test_extraction.py

@@ -873,7 +873,7 @@
 &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">"XX Large"</ins>
 </span>
 <div data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;generated&quot;: false,
-&quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}"">Price: 45</div>
+&quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">Price: 45</div>
 """
 
 EXTRACT_PAGE28 = u"""
@@ -891,6 +891,29 @@
 <div>Price: 45</div>
 """
 
+ANNOTATED_PAGE29 = u"""
+<table>
+<tr><td data-scrapy-annotate="{&quot;variant&quot;: 1, &quot;generated&quot;: false,
+ &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">Name 1</td><td data-scrapy-annotate="{&quot;variant&quot;: 1, &quot;generated&quot;: false,
+ &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">Desc 1</td><td><span data-scrapy-annotate="{&quot;variant&quot;: 1, &quot;generated&quot;: false,
+ &quot;annotations&quot;: {&quot;content&quot;: &quot;tag&quot;}}">Tag 1</span><span>Tag2</span><span data-scrapy-annotate="{&quot;variant&quot;: 1, &quot;generated&quot;: false,
+ &quot;annotations&quot;: {&quot;content&quot;: &quot;tag&quot;}}">Tag 3</span></td></tr>
+<tr><td>Name 2</td><td>Desc 2</td><td><span>Tag 7</span><span>Tag 8</span></span>Tag 9</span></td></tr>
+<tr><td data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;generated&quot;: false,
+ &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">Name 3</td><td data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;generated&quot;: false,
+ &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">Desc 3</td><td><span data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;generated&quot;: false,
+ &quot;annotations&quot;: {&quot;content&quot;: &quot;tag&quot;}}">Tag 4</span><span>Tag5</span><span data-scrapy-annotate="{&quot;variant&quot;: 2, &quot;generated&quot;: false,
+ &quot;annotations&quot;: {&quot;content&quot;: &quot;tag&quot;}}">Tag 6</span></td></tr>
+</table>
+"""
+
+EXTRACT_PAGE29 = u"""
+<table>
+<tr><td>Name 1</td><td>Desc 1</td><td><span>Tag 1</span><span>Tag 2</span><span>Tag 3</span></td></tr>
+<tr><td>Name 2</td><td>Desc 2</td><td><span>Tag 4</span><span>Tag 5</span><span>Tag 6</span></td></tr>
+<tr><td>Name 3</td><td>Desc 3</td><td><span>Tag 7</span><span>Tag 8</span><span>Tag 9</span></td></tr>
+</table>
+"""
 
 DEFAULT_DESCRIPTOR = ItemDescriptor('test', 
         'item test, removes tags from description attribute',
@@ -1147,6 +1170,14 @@
                     {"name": ['"XX Large"'], "site_id": ["46"]}]
             }
     ),
+    ('repeated annotation inside variants', [ANNOTATED_PAGE29], EXTRACT_PAGE29, DEFAULT_DESCRIPTOR, 
+            {'variants': [
+                {u'tag': [u'Tag 1', u'Tag 2', u'Tag 3'], u'description': [u'Desc 1'], u'name': [u'Name 1']},
+                {u'tag': [u'Tag 4', u'Tag 5', u'Tag 6'], u'description': [u'Desc 2'], u'name': [u'Name 2']},
+                {u'tag': [u'Tag 7', u'Tag 8', u'Tag 9'], u'description': [u'Desc 3'], u'name': [u'Name 3']}]
+            }
+
+    ),
 ]
 
 class TestIbl(TestCase):