Merge pull request #46 from kmike/unicode-support

[MRG] scrapely.tool: add support for non-ascii <text> and <data> arguments

Author: pablohoffman

scrapely/tool.py

@@ -43,21 +43,21 @@ def do_t(self, line):
         """t <template> <text> - test selection text"""
         template_id, criteria = line.split(' ', 1)
         t = self._load_template(template_id)
-        criteria = parse_criteria(criteria)
+        criteria = self._parse_criteria(criteria)
         tm = TemplateMaker(t)
         selection = apply_criteria(criteria, tm)
         for n, i in enumerate(selection):
             print "[%d] %r" % (n, remove_annotation(tm.selected_data(i)))
 
     def do_a(self, line):
         """a <template> <data> [-n number] [-f field]- add or test annotation
-        
+
         Add a new annotation (if -f is passed) or test what would be annotated
         otherwise
         """
         template_id, criteria = line.split(' ', 1)
         t = self._load_template(template_id)
-        criteria = parse_criteria(criteria)
+        criteria = self._parse_criteria(criteria)
         tm = TemplateMaker(t)
         selection = apply_criteria(criteria, tm)
         if criteria.field:
@@ -78,7 +78,7 @@ def do_al(self, template_id):
         t = self._load_template(template_id)
         tm = TemplateMaker(t)
         for n, (a, i) in enumerate(tm.annotations()):
-            print "[%s-%d] (%s) %r" % (template_id, n, a['annotations']['content'], 
+            print "[%s-%d] (%s) %r" % (template_id, n, a['annotations']['content'],
                 remove_annotation(tm.selected_data(i)))
 
     def do_s(self, url):
@@ -126,21 +126,24 @@ def _save_templates(self, templates):
         with open(self.filename, 'w') as f:
             templates = [page_to_dict(t) for t in templates]
             return json.dump({'templates': templates}, f)
-        
+
+    def _parse_criteria(self, criteria_str):
+        """Parse the given criteria string and returns a criteria object"""
+        p = optparse.OptionParser()
+        p.add_option('-f', '--field', help='field to annotate')
+        p.add_option('-n', '--number', type="int", help='number of result to select')
+        o, a = p.parse_args(shlex.split(criteria_str))
+
+        encoding = getattr(self.stdin, 'encoding', None) or sys.stdin.encoding
+        o.text = ' '.join(a).decode(encoding or 'ascii')
+        return o
+
+
 def parse_at(ta_line):
     p = optparse.OptionParser()
     p.add_option('-e', '--encoding', help='page encoding')
     return p.parse_args(shlex.split(ta_line))
 
-def parse_criteria(criteria_str):
-    """Parse the given criteria string and returns a criteria object"""
-    p = optparse.OptionParser()
-    p.add_option('-f', '--field', help='field to annotate')
-    p.add_option('-n', '--number', type="int", help='number of result to select')
-    o, a = p.parse_args(shlex.split(criteria_str))
-    o.text = ' '.join(a)
-    return o
-
 def apply_criteria(criteria, tm):
     """Apply the given criteria object to the given template"""
     func = best_match(criteria.text) if criteria.text else lambda x, y: False