Add python3 support for url_to_page

Author: ruairif

NEWS

@@ -1,6 +1,11 @@
 Scrapely release notes
 ======================
 
+0.13.2 (2016-12-21)
+-------------------
+
+- Add python3 support for `url_to_page` function
+
 0.13.1 (2016-12-21)
 -------------------
 

scrapely/htmlpage.py

@@ -187,7 +187,7 @@ def url_to_page(url, encoding=None, default_encoding='utf-8'):
     if encoding is None:
         try:
             # Python 3.x
-            content_type_header = fh.getheader("content-type")
+            content_type_header = fh.headers.get("content-type")
         except AttributeError:
             # Python 2.x
             content_type_header = info.getheader("content-type")

tests/test_htmlpage.py

@@ -1,35 +1,43 @@
 """
 htmlpage.py tests
 """
+import os
 import copy
+import json
 from unittest import TestCase
 
-from scrapely.htmlpage import parse_html, HtmlTag, HtmlDataFragment, HtmlPage
+from scrapely.htmlpage import (
+    parse_html, HtmlTag, HtmlDataFragment, HtmlPage, url_to_page
+)
 from .test_htmlpage_data import *
 from . import iter_samples
+BASE_PATH = os.path.abspath(os.path.dirname(__file__))
+
 
 def _encode_element(el):
     """
     jsonize parse element
     """
     if isinstance(el, HtmlTag):
         return {"tag": el.tag, "attributes": el.attributes,
-            "start": el.start, "end": el.end, "tag_type": el.tag_type}
+                "start": el.start, "end": el.end, "tag_type": el.tag_type}
     if isinstance(el, HtmlDataFragment):
         return {"start": el.start, "end": el.end, "is_text_content": el.is_text_content}
     raise TypeError
 
+
 def _decode_element(dct):
     """
     dejsonize parse element
     """
     if "tag" in dct:
-        return HtmlTag(dct["tag_type"], dct["tag"], \
-            dct["attributes"], dct["start"], dct["end"])
+        return HtmlTag(dct["tag_type"], dct["tag"],
+                       dct["attributes"], dct["start"], dct["end"])
     if "start" in dct:
         return HtmlDataFragment(dct["start"], dct["end"], dct.get("is_text_content", True))
     return dct
 
+
 class TestParseHtml(TestCase):
     """Test for parse_html"""
     def _test_sample(self, source, expected_parsed, samplecount=None):
@@ -165,3 +173,12 @@ def test_copy(self):
         self.assertEqual(regiondeepcopy.end_index, 15)
         self.assertFalse(region is regiondeepcopy)
         self.assertFalse(region.htmlpage is regiondeepcopy.htmlpage)
+
+    def test_load_page_from_url(self):
+        filepath = os.path.join(BASE_PATH, 'samples/samples_htmlpage_0')
+        url = 'file://{}.{}'.format(filepath, 'html')
+        page = url_to_page(url)
+        parsed = json.load(open('{}.{}'.format(filepath, 'json')))
+        parsed = [_decode_element(d) for d in parsed]
+        self.assertEqual(page.url, url)
+        self._test_sample(page.body, parsed, 1)