scrapely.tool: add support for non-ascii <text> and <data> arguments

<text> and <data> arguments are parsed by parse_criteria function
(it uses shlex and optparse for parsing). Data that is passed to parse_criteria
function is extracted from "line" argument of do_<…> methods.
This "line" argument is read from self.stdin by cmd.Cmd and
passed to do_ methods. In Python 2.x sys.stdin (which is
default for cmd.Cmd.stdin) is binary, so "line" is a bytestring;
its encoding is self.stdin.encoding. That's why <text> and <data>
argument values was previously bytestrings; when passed to
other scrapely functions they eventually got implicitly decoded
using sys.getdefaultencoding() - this usually leads to
UnicodeDecodeError if input text is non-ascii.

The fix is to decode these arguments using self.stdin.encoding
before passing them to scrapely. This is done after shlex call
because shlex doesn't support unicode. Non-ascii "field" arguments
are still unsupported.

Author: kmike

scrapely/tool.py

@@ -43,7 +43,7 @@ def do_t(self, line):
         """t <template> <text> - test selection text"""
         template_id, criteria = line.split(' ', 1)
         t = self._load_template(template_id)
-        criteria = parse_criteria(criteria)
+        criteria = self._parse_criteria(criteria)
         tm = TemplateMaker(t)
         selection = apply_criteria(criteria, tm)
         for n, i in enumerate(selection):
@@ -57,7 +57,7 @@ def do_a(self, line):
         """
         template_id, criteria = line.split(' ', 1)
         t = self._load_template(template_id)
-        criteria = parse_criteria(criteria)
+        criteria = self._parse_criteria(criteria)
         tm = TemplateMaker(t)
         selection = apply_criteria(criteria, tm)
         if criteria.field:
@@ -126,21 +126,22 @@ def _save_templates(self, templates):
         with open(self.filename, 'w') as f:
             templates = [page_to_dict(t) for t in templates]
             return json.dump({'templates': templates}, f)
-        
+
+    def _parse_criteria(self, criteria_str):
+        """Parse the given criteria string and returns a criteria object"""
+        p = optparse.OptionParser()
+        p.add_option('-f', '--field', help='field to annotate')
+        p.add_option('-n', '--number', type="int", help='number of result to select')
+        o, a = p.parse_args(shlex.split(criteria_str))
+        o.text = ' '.join(a).decode(self.stdin.encoding or 'ascii')
+        return o
+
+
 def parse_at(ta_line):
     p = optparse.OptionParser()
     p.add_option('-e', '--encoding', help='page encoding')
     return p.parse_args(shlex.split(ta_line))
 
-def parse_criteria(criteria_str):
-    """Parse the given criteria string and returns a criteria object"""
-    p = optparse.OptionParser()
-    p.add_option('-f', '--field', help='field to annotate')
-    p.add_option('-n', '--number', type="int", help='number of result to select')
-    o, a = p.parse_args(shlex.split(criteria_str))
-    o.text = ' '.join(a)
-    return o
-
 def apply_criteria(criteria, tm):
     """Apply the given criteria object to the given template"""
     func = best_match(criteria.text) if criteria.text else lambda x, y: False