|
11 | 11 | from six.moves.urllib.request import urlopen |
12 | 12 | from copy import deepcopy |
13 | 13 | from w3lib.encoding import html_to_unicode |
14 | | - |
15 | | -from . import _htmlpage |
16 | | - |
17 | | - |
18 | | -parse_html = _htmlpage.parse_html |
19 | | -HtmlDataFragment = _htmlpage.HtmlDataFragment |
20 | | -HtmlTag = _htmlpage.HtmlTag |
21 | | -HtmlTagType = _htmlpage.HtmlTagType |
| 14 | +try: |
| 15 | + from . import _htmlpage |
| 16 | + parse_html = _htmlpage.parse_html |
| 17 | + HtmlDataFragment = _htmlpage.HtmlDataFragment |
| 18 | + HtmlTag = _htmlpage.HtmlTag |
| 19 | + HtmlTagType = _htmlpage.HtmlTagType |
| 20 | +except ImportError: |
| 21 | + import re |
| 22 | + from collections import OrderedDict |
| 23 | + |
| 24 | + class HtmlTagType(object): |
| 25 | + OPEN_TAG = 1 |
| 26 | + CLOSE_TAG = 2 |
| 27 | + UNPAIRED_TAG = 3 |
| 28 | + |
| 29 | + class HtmlDataFragment(object): |
| 30 | + __slots__ = ('start', 'end', 'is_text_content') |
| 31 | + |
| 32 | + def __init__(self, start, end, is_text_content=False): |
| 33 | + self.start = start |
| 34 | + self.end = end |
| 35 | + self.is_text_content = is_text_content |
| 36 | + |
| 37 | + def __str__(self): |
| 38 | + return "<HtmlDataFragment [%s:%s] is_text_content: %s>" % ( |
| 39 | + self.start, self.end, self.is_text_content) |
| 40 | + |
| 41 | + def __repr__(self): |
| 42 | + return str(self) |
| 43 | + |
| 44 | + class HtmlTag(HtmlDataFragment): |
| 45 | + __slots__ = ('tag_type', 'tag', '_attributes', '_attr_text') |
| 46 | + |
| 47 | + def __init__(self, tag_type, tag, attr_text, start, end): |
| 48 | + HtmlDataFragment.__init__(self, start, end) |
| 49 | + self.tag_type = tag_type |
| 50 | + self.tag = tag |
| 51 | + if isinstance(attr_text, dict): |
| 52 | + self._attributes = attr_text |
| 53 | + self._attr_text = None |
| 54 | + else: # defer loading attributes until necessary |
| 55 | + self._attributes = OrderedDict() |
| 56 | + self._attr_text = attr_text |
| 57 | + |
| 58 | + @property |
| 59 | + def attributes(self): |
| 60 | + if not self._attributes and self._attr_text: |
| 61 | + for attr_match in _ATTR_REGEXP.findall(self._attr_text): |
| 62 | + name = attr_match[0].lower() |
| 63 | + values = [v for v in attr_match[1:] if v] |
| 64 | + # According to HTML spec if attribute name is repeated only |
| 65 | + # the first one is taken into account |
| 66 | + if name not in self._attributes: |
| 67 | + self._attributes[name] = values[0] if values else None |
| 68 | + return self._attributes |
| 69 | + |
| 70 | + def __str__(self): |
| 71 | + attributes = ', '.join( |
| 72 | + sorted(["%s: %s" % (k, repr(v)) |
| 73 | + for k, v in self.attributes.items()])) |
| 74 | + return "<HtmlTag tag='%s' attributes={%s} type='%d' [%s:%s]>" % ( |
| 75 | + self.tag, attributes, self.tag_type, self.start, self.end) |
| 76 | + |
| 77 | + def __repr__(self): |
| 78 | + return str(self) |
| 79 | + |
| 80 | + _ATTR = ("((?:[^=/<>\s]|/(?!>))+)(?:\s*=(?:\s*\"(.*?)\"|\s*'(.*?)'|" |
| 81 | + "([^>\s]+))?)?") |
| 82 | + _TAG = "<(\/?)(\w+(?::\w+)?)((?:\s*" + _ATTR + ")+\s*|\s*)(\/?)>?" |
| 83 | + _DOCTYPE = r"<!DOCTYPE.*?>" |
| 84 | + _SCRIPT = "(<script.*?>)(.*?)(</script.*?>)" |
| 85 | + _COMMENT = "(<!--.*?-->|<\?.+?>)" |
| 86 | + |
| 87 | + _ATTR_REGEXP = re.compile(_ATTR, re.I | re.DOTALL) |
| 88 | + _HTML_REGEXP = re.compile("%s|%s|%s" % (_COMMENT, _SCRIPT, _TAG), |
| 89 | + re.I | re.DOTALL) |
| 90 | + _DOCTYPE_REGEXP = re.compile("(?:%s)" % _DOCTYPE) |
| 91 | + _COMMENT_REGEXP = re.compile(_COMMENT, re.DOTALL) |
| 92 | + |
| 93 | + def parse_html(text): |
| 94 | + """Higher level html parser. Calls lower level parsers and joins sucesive |
| 95 | + HtmlDataFragment elements in a single one. |
| 96 | + """ |
| 97 | + # If have doctype remove it. |
| 98 | + start_pos = 0 |
| 99 | + match = _DOCTYPE_REGEXP.match(text) |
| 100 | + if match: |
| 101 | + start_pos = match.end() |
| 102 | + prev_end = start_pos |
| 103 | + for match in _HTML_REGEXP.finditer(text, start_pos): |
| 104 | + start = match.start() |
| 105 | + end = match.end() |
| 106 | + |
| 107 | + if start > prev_end: |
| 108 | + yield HtmlDataFragment(prev_end, start, True) |
| 109 | + |
| 110 | + if match.groups()[0] is not None: # comment |
| 111 | + yield HtmlDataFragment(start, end) |
| 112 | + elif match.groups()[1] is not None: # <script>...</script> |
| 113 | + for e in _parse_script(match): |
| 114 | + yield e |
| 115 | + else: # tag |
| 116 | + yield _parse_tag(match) |
| 117 | + prev_end = end |
| 118 | + textlen = len(text) |
| 119 | + if prev_end < textlen: |
| 120 | + yield HtmlDataFragment(prev_end, textlen, True) |
| 121 | + |
| 122 | + def _parse_script(match): |
| 123 | + """parse a <script>...</script> region matched by _HTML_REGEXP""" |
| 124 | + open_text, content, close_text = match.groups()[1:4] |
| 125 | + |
| 126 | + open_tag = _parse_tag(_HTML_REGEXP.match(open_text)) |
| 127 | + open_tag.start = match.start() |
| 128 | + open_tag.end = match.start() + len(open_text) |
| 129 | + |
| 130 | + close_tag = _parse_tag(_HTML_REGEXP.match(close_text)) |
| 131 | + close_tag.start = match.end() - len(close_text) |
| 132 | + close_tag.end = match.end() |
| 133 | + |
| 134 | + yield open_tag |
| 135 | + if open_tag.end < close_tag.start: |
| 136 | + start_pos = 0 |
| 137 | + for m in _COMMENT_REGEXP.finditer(content): |
| 138 | + if m.start() > start_pos: |
| 139 | + yield HtmlDataFragment( |
| 140 | + open_tag.end + start_pos, open_tag.end + m.start()) |
| 141 | + yield HtmlDataFragment( |
| 142 | + open_tag.end + m.start(), open_tag.end + m.end()) |
| 143 | + start_pos = m.end() |
| 144 | + if open_tag.end + start_pos < close_tag.start: |
| 145 | + yield HtmlDataFragment( |
| 146 | + open_tag.end + start_pos, close_tag.start) |
| 147 | + yield close_tag |
| 148 | + |
| 149 | + def _parse_tag(match): |
| 150 | + """ |
| 151 | + parse a tag matched by _HTML_REGEXP |
| 152 | + """ |
| 153 | + data = match.groups() |
| 154 | + closing, tag, attr_text = data[4:7] |
| 155 | + # if tag is None then the match is a comment |
| 156 | + if tag is not None: |
| 157 | + unpaired = data[-1] |
| 158 | + if closing: |
| 159 | + tag_type = HtmlTagType.CLOSE_TAG |
| 160 | + elif unpaired: |
| 161 | + tag_type = HtmlTagType.UNPAIRED_TAG |
| 162 | + else: |
| 163 | + tag_type = HtmlTagType.OPEN_TAG |
| 164 | + return HtmlTag(tag_type, tag.lower(), attr_text, match.start(), |
| 165 | + match.end()) |
22 | 166 |
|
23 | 167 |
|
24 | 168 | def url_to_page(url, encoding=None, default_encoding='utf-8'): |
@@ -164,6 +308,7 @@ def __new__(cls, htmlpage, start_index, end_index): |
164 | 308 | text_start = htmlpage.parsed_body[start_index].start |
165 | 309 | text_end = htmlpage.parsed_body[end_index or -1].end |
166 | 310 | text = htmlpage.body[text_start:text_end] |
| 311 | + |
167 | 312 | return HtmlPageRegion.__new__(cls, htmlpage, text) |
168 | 313 |
|
169 | 314 | def __init__(self, htmlpage, start_index, end_index): |
|
0 commit comments