From f2d780edd52d6a83c6222df7eebfaa1dd54b55b1 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Thu, 3 Oct 2013 04:25:56 +0600
Subject: [PATCH 1/2] scrapely.tool: add support for non-ascii <text> and
 <data> arguments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

<text> and <data> arguments are parsed by parse_criteria function
(it uses shlex and optparse for parsing). Data that is passed to parse_criteria
function is extracted from "line" argument of do_<…> methods.
This "line" argument is read from self.stdin by cmd.Cmd and
passed to do_ methods. In Python 2.x sys.stdin (which is
default for cmd.Cmd.stdin) is binary, so "line" is a bytestring;
its encoding is self.stdin.encoding. That's why <text> and <data>
argument values was previously bytestrings; when passed to
other scrapely functions they eventually got implicitly decoded
using sys.getdefaultencoding() - this usually leads to
UnicodeDecodeError if input text is non-ascii.

The fix is to decode these arguments using self.stdin.encoding
before passing them to scrapely. This is done after shlex call
because shlex doesn't support unicode. Non-ascii "field" arguments
are still unsupported.
---
 scrapely/tool.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/scrapely/tool.py b/scrapely/tool.py
index 172c695..6a5b23e 100644
--- a/scrapely/tool.py
+++ b/scrapely/tool.py
@@ -43,7 +43,7 @@ def do_t(self, line):
         """t <template> <text> - test selection text"""
         template_id, criteria = line.split(' ', 1)
         t = self._load_template(template_id)
-        criteria = parse_criteria(criteria)
+        criteria = self._parse_criteria(criteria)
         tm = TemplateMaker(t)
         selection = apply_criteria(criteria, tm)
         for n, i in enumerate(selection):
@@ -57,7 +57,7 @@ def do_a(self, line):
         """
         template_id, criteria = line.split(' ', 1)
         t = self._load_template(template_id)
-        criteria = parse_criteria(criteria)
+        criteria = self._parse_criteria(criteria)
         tm = TemplateMaker(t)
         selection = apply_criteria(criteria, tm)
         if criteria.field:
@@ -126,21 +126,22 @@ def _save_templates(self, templates):
         with open(self.filename, 'w') as f:
             templates = [page_to_dict(t) for t in templates]
             return json.dump({'templates': templates}, f)
-        
+
+    def _parse_criteria(self, criteria_str):
+        """Parse the given criteria string and returns a criteria object"""
+        p = optparse.OptionParser()
+        p.add_option('-f', '--field', help='field to annotate')
+        p.add_option('-n', '--number', type="int", help='number of result to select')
+        o, a = p.parse_args(shlex.split(criteria_str))
+        o.text = ' '.join(a).decode(self.stdin.encoding or 'ascii')
+        return o
+
+
 def parse_at(ta_line):
     p = optparse.OptionParser()
     p.add_option('-e', '--encoding', help='page encoding')
     return p.parse_args(shlex.split(ta_line))
 
-def parse_criteria(criteria_str):
-    """Parse the given criteria string and returns a criteria object"""
-    p = optparse.OptionParser()
-    p.add_option('-f', '--field', help='field to annotate')
-    p.add_option('-n', '--number', type="int", help='number of result to select')
-    o, a = p.parse_args(shlex.split(criteria_str))
-    o.text = ' '.join(a)
-    return o
-
 def apply_criteria(criteria, tm):
     """Apply the given criteria object to the given template"""
     func = best_match(criteria.text) if criteria.text else lambda x, y: False

From 4e439343d1ea70ceb25b8ab2c0bc486421fecb59 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Fri, 4 Oct 2013 07:02:00 +0600
Subject: [PATCH 2/2] fix command-line scrapely.tool usage that was broken by
 previous commit

---
 scrapely/tool.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scrapely/tool.py b/scrapely/tool.py
index 6a5b23e..5f57cba 100644
--- a/scrapely/tool.py
+++ b/scrapely/tool.py
@@ -51,7 +51,7 @@ def do_t(self, line):
 
     def do_a(self, line):
         """a <template> <data> [-n number] [-f field]- add or test annotation
-        
+
         Add a new annotation (if -f is passed) or test what would be annotated
         otherwise
         """
@@ -78,7 +78,7 @@ def do_al(self, template_id):
         t = self._load_template(template_id)
         tm = TemplateMaker(t)
         for n, (a, i) in enumerate(tm.annotations()):
-            print "[%s-%d] (%s) %r" % (template_id, n, a['annotations']['content'], 
+            print "[%s-%d] (%s) %r" % (template_id, n, a['annotations']['content'],
                 remove_annotation(tm.selected_data(i)))
 
     def do_s(self, url):
@@ -133,7 +133,9 @@ def _parse_criteria(self, criteria_str):
         p.add_option('-f', '--field', help='field to annotate')
         p.add_option('-n', '--number', type="int", help='number of result to select')
         o, a = p.parse_args(shlex.split(criteria_str))
-        o.text = ' '.join(a).decode(self.stdin.encoding or 'ascii')
+
+        encoding = getattr(self.stdin, 'encoding', None) or sys.stdin.encoding
+        o.text = ' '.join(a).decode(encoding or 'ascii')
         return o