Fast get_docid for microdata parser

Previously, get_docid had quadratic complexity, as for each node, it
counted it's position in the tree iterating over all previous nodes.
Here we fix this by computing get_docid in the same way for the first
element, and then incrementing it by one, which gives the same result
but has linear complexity and is much faster (e.g. bringing time from
190+ s to 6 s on example from GH-147).
Other discarded implementation options:
- using some other identifier for the node - this would break backwards
  compatibility, as this property was exposed and documented at least in
  the code
- instead of calling get_docid for the first element, just iterate over
  the whole document (from root) to build ids - this is less
  efficient in case we call extraction on a part of the document.

Author: lopuhin

extruct/w3cmicrodata.py

@@ -45,6 +45,7 @@
 
 
 class LxmlMicrodataExtractor(object):
+    # iterate in document order (used below for get_docid optimization)
     _xp_item = lxml.etree.XPath('descendant-or-self::*[@itemscope]')
     _xp_prop = lxml.etree.XPath("""set:difference(.//*[@itemprop],
                                                   .//*[@itemscope]//*[@itemprop])""",
@@ -62,23 +63,47 @@ def __init__(self, nested=True, strict=False, add_text_content=False, add_html_n
         self.add_text_content = add_text_content
         self.add_html_node = add_html_node
 
-    def get_docid(self, node):
-        return int(self._xp_item_docid(node))
+    def get_docid(self, node, itemids):
+        try:
+            return itemids[node]  # same as self.get_docid(node, {})
+        except KeyError:
+            # Even after itemids are built,
+            # this might fail if extract_items is called on a part of the document,
+            # and then properties reference some node which is not in itemids,
+            # although this does not look likely in practice,
+            # so not a performance concern.
+            return int(self._xp_item_docid(node))
 
     def extract(self, htmlstring, base_url=None, encoding="UTF-8"):
         tree = parse_html(htmlstring, encoding=encoding)
         return self.extract_items(tree, base_url)
 
     def extract_items(self, document, base_url):
+        itemids = self._build_itemids(document)
         items_seen = set()
         return [
             item for item in (
-                self._extract_item(it, items_seen=items_seen, base_url=base_url)
+                self._extract_item(
+                    it, items_seen=items_seen, base_url=base_url, itemids=itemids)
                 for it in self._xp_item(document))
             if item]
 
-    def _extract_item(self, node, items_seen, base_url):
-        itemid = self.get_docid(node)
+    def _build_itemids(self, document):
+        itemid = None
+        itemids = {}
+        for node in self._xp_item(document):
+            if itemid is None:
+                itemid = self.get_docid(node, {})
+                assert itemid is not None
+            else:
+                # this is the same as self.get_docid(node) but faster,
+                # calling get_docid on each iteration leads to quadratic complexity
+                itemid += 1
+            itemids[node] = itemid
+        return itemids
+
+    def _extract_item(self, node, items_seen, base_url, itemids):
+        itemid = self.get_docid(node, itemids)
 
         if self.nested:
             if itemid in items_seen:
@@ -95,21 +120,22 @@ def _extract_item(self, node, items_seen, base_url):
             else:
                 item["type"] = types
 
-            itemid = node.get('itemid')
-            if itemid:
-                item["id"] = itemid.strip()
+            nodeid = node.get('itemid')
+            if nodeid:
+                item["id"] = nodeid.strip()
 
         properties = collections.defaultdict(list)
         for name, value in self._extract_properties(
-                node, items_seen=items_seen, base_url=base_url):
+                node, items_seen=items_seen, base_url=base_url, itemids=itemids):
             properties[name].append(value)
 
         # process item references
         refs = node.get('itemref', '').split()
         if refs:
             for refid in refs:
                 for name, value in self._extract_property_refs(
-                        node, refid, items_seen=items_seen, base_url=base_url):
+                        node, refid, items_seen=items_seen, base_url=base_url,
+                        itemids=itemids):
                     properties[name].append(value)
 
         props = []
@@ -123,7 +149,8 @@ def _extract_item(self, node, items_seen, base_url):
         else:
             # item without properties; let's use the node itself
             item["value"] = self._extract_property_value(
-                node, force=True, items_seen=items_seen, base_url=base_url)
+                node, force=True, items_seen=items_seen, base_url=base_url,
+                itemids=itemids)
 
         # below are not in the specs, but can be handy
         if self.add_text_content:
@@ -135,19 +162,19 @@ def _extract_item(self, node, items_seen, base_url):
 
         return item
 
-    def _extract_properties(self, node, items_seen, base_url):
+    def _extract_properties(self, node, items_seen, base_url, itemids):
         for prop in self._xp_prop(node):
             for p, v in self._extract_property(
-                    prop, items_seen=items_seen, base_url=base_url):
+                    prop, items_seen=items_seen, base_url=base_url, itemids=itemids):
                 yield p, v
 
-    def _extract_property_refs(self, node, refid, items_seen, base_url):
+    def _extract_property_refs(self, node, refid, items_seen, base_url, itemids):
         ref_node = node.xpath("id($refid)[1]", refid=refid)
         if not ref_node:
             return
         ref_node = ref_node[0]
         extract_fn = partial(self._extract_property, items_seen=items_seen,
-                             base_url=base_url)
+                             base_url=base_url, itemids=itemids)
         if 'itemprop' in ref_node.keys() and 'itemscope' in ref_node.keys():
             # An full item will be extracted from the node, no need to look
             # for individual properties in childs
@@ -162,20 +189,20 @@ def _extract_property_refs(self, node, refid, items_seen, base_url):
                     for p, v in extract_fn(prop):
                         yield p, v
 
-    def _extract_property(self, node, items_seen, base_url):
+    def _extract_property(self, node, items_seen, base_url, itemids):
         props = node.get("itemprop").split()
         value = self._extract_property_value(
-            node, items_seen=items_seen, base_url=base_url)
+            node, items_seen=items_seen, base_url=base_url, itemids=itemids)
         return [(p, value) for p in props]
 
-    def _extract_property_value(self, node, items_seen, base_url, force=False):
+    def _extract_property_value(self, node, items_seen, base_url, itemids, force=False):
         #http://www.w3.org/TR/microdata/#values
         if not force and node.get("itemscope") is not None:
             if self.nested:
                 return self._extract_item(
-                    node, items_seen=items_seen, base_url=base_url)
+                    node, items_seen=items_seen, base_url=base_url, itemids=itemids)
             else:
-                return {"iid_ref": self.get_docid(node)}
+                return {"iid_ref": self.get_docid(node, itemids)}
 
         elif node.tag == "meta":
             return node.get("content", "")