From 0135002b201c12ade5676cbba90902525caff9f8 Mon Sep 17 00:00:00 2001
From: xyb <xieyanbo@gmail.com>
Date: Tue, 10 Sep 2013 13:00:56 +0800
Subject: [PATCH 1/3] support CJK string annotation; print readably CJK string
 in scrapely.tool's output

---
 scrapely/template.py |  2 +-
 scrapely/tool.py     | 48 ++++++++++++++++++++++++++++++++++++++------
 2 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/scrapely/template.py b/scrapely/template.py
index 33ae2e1..5c96b8f 100644
--- a/scrapely/template.py
+++ b/scrapely/template.py
@@ -92,7 +92,7 @@ def best_match(text):
     """Function to use in TemplateMaker.annotate()"""
     def func(fragment, page):
         fdata = page.fragment_data(fragment).strip()
-        if text in fdata:
+        if text.decode('utf8') in fdata:
             return float(len(text)) / len(fdata) - (1e-6 * fragment.start)
         else:
             return 0.0
diff --git a/scrapely/tool.py b/scrapely/tool.py
index 172c695..5b87403 100644
--- a/scrapely/tool.py
+++ b/scrapely/tool.py
@@ -6,6 +6,19 @@
 from scrapely.template import TemplateMaker, best_match
 from scrapely.extraction import InstanceBasedLearningExtractor
 
+REPR_UNICODE_CHAR = re.compile(r'(?<!\\)(\\u[0-9a-f]{4,4})')
+
+
+def readable_repr(obj):
+    '''Return printing-friendly unicode repr string
+    '''
+    def replace_unicode_char(repr_char):
+        return unichr(int(str(repr_char.group())[2:], base=16))
+
+    repr_string = repr(obj)
+    return REPR_UNICODE_CHAR.sub(replace_unicode_char, repr_string)
+
+
 class IblTool(cmd.Cmd):
 
     prompt = 'scrapely> '
@@ -17,6 +30,8 @@ def __init__(self, filename, **kw):
     def do_ta(self, line):
         """ta <url> [--encoding ENCODING] - add template"""
         opts, (url,) = parse_at(line)
+        if assert_or_print(url, "missing url"):
+            return
         t = url_to_page(url, opts.encoding)
         templates = self._load_templates()
         templates.append(t)
@@ -31,7 +46,11 @@ def do_tl(self, line):
 
     def do_td(self, template_id):
         """dt <template> - delete template"""
+        if assert_or_print(template_id, "missing template id"):
+            return
         templates = self._load_templates()
+        if assert_or_print(teamplates, "no templates available"):
+            return
         try:
             del templates[int(template_id)]
             self._save_templates(templates)
@@ -41,13 +60,20 @@ def do_td(self, template_id):
 
     def do_t(self, line):
         """t <template> <text> - test selection text"""
+        if assert_or_print(line, "missing template id or selection text"):
+            return
+        if assert_or_print(' ' in line, "missing template id or selection text"):
+            return
         template_id, criteria = line.split(' ', 1)
         t = self._load_template(template_id)
+        if assert_or_print(t, "template not found: %s" % template_id):
+            return
         criteria = parse_criteria(criteria)
         tm = TemplateMaker(t)
         selection = apply_criteria(criteria, tm)
         for n, i in enumerate(selection):
-            print "[%d] %r" % (n, remove_annotation(tm.selected_data(i)))
+            print "[%d] %s" % (n,
+                               readable_repr(remove_annotation(tm.selected_data(i))))
 
     def do_a(self, line):
         """a <template> <data> [-n number] [-f field]- add or test annotation
@@ -55,8 +81,14 @@ def do_a(self, line):
         Add a new annotation (if -f is passed) or test what would be annotated
         otherwise
         """
+        if assert_or_print(line, "missing template id and selection text"):
+            return
+        if assert_or_print(' ' in line, "missing template id or selection text"):
+            return
         template_id, criteria = line.split(' ', 1)
         t = self._load_template(template_id)
+        if assert_or_print(t, "template not found: %s" % template_id):
+            return
         criteria = parse_criteria(criteria)
         tm = TemplateMaker(t)
         selection = apply_criteria(criteria, tm)
@@ -65,27 +97,31 @@ def do_a(self, line):
                 index = selection[0]
                 tm.annotate_fragment(index, criteria.field)
                 self._save_template(template_id, tm.get_template())
-                print "[new] (%s) %r" % (criteria.field,
-                    remove_annotation(tm.selected_data(index)))
+                print "[new] (%s) %s" % (criteria.field,
+                    readable_repr(remove_annotation(tm.selected_data(index))))
         else:
             for n, i in enumerate(selection):
-                print "[%d] %r" % (n, remove_annotation(tm.selected_data(i)))
+                print "[%d] %s" % (n, readable_repr(remove_annotation(tm.selected_data(i))))
 
     def do_al(self, template_id):
         """al <template> - list annotations"""
         if assert_or_print(template_id, "missing template id"):
             return
         t = self._load_template(template_id)
+        if assert_or_print(t, "template not found: %s" % template_id):
+            return
         tm = TemplateMaker(t)
         for n, (a, i) in enumerate(tm.annotations()):
-            print "[%s-%d] (%s) %r" % (template_id, n, a['annotations']['content'], 
-                remove_annotation(tm.selected_data(i)))
+            print "[%s-%d] (%s) %s" % (template_id, n, a['annotations']['content'], 
+                readable_repr(remove_annotation(tm.selected_data(i))))
 
     def do_s(self, url):
         """s <url> - scrape url"""
         templates = self._load_templates()
         if assert_or_print(templates, "no templates available"):
             return
+        if assert_or_print(url, "missing url"):
+            return
         # fall back to the template encoding if none is specified
         page = url_to_page(url, default_encoding=templates[0].encoding)
         ex = InstanceBasedLearningExtractor((t, None) for t in templates)

From 8984924bcd837882e8739e5b7bb9d2218b57e9ac Mon Sep 17 00:00:00 2001
From: xyb <xieyanbo@gmail.com>
Date: Sun, 29 Sep 2013 15:48:36 +0800
Subject: [PATCH 2/3] fix text encoding when calling best_match

---
 scrapely/template.py | 2 +-
 scrapely/tool.py     | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapely/template.py b/scrapely/template.py
index 5c96b8f..33ae2e1 100644
--- a/scrapely/template.py
+++ b/scrapely/template.py
@@ -92,7 +92,7 @@ def best_match(text):
     """Function to use in TemplateMaker.annotate()"""
     def func(fragment, page):
         fdata = page.fragment_data(fragment).strip()
-        if text.decode('utf8') in fdata:
+        if text in fdata:
             return float(len(text)) / len(fdata) - (1e-6 * fragment.start)
         else:
             return 0.0
diff --git a/scrapely/tool.py b/scrapely/tool.py
index 5b87403..5b6bb02 100644
--- a/scrapely/tool.py
+++ b/scrapely/tool.py
@@ -179,7 +179,10 @@ def parse_criteria(criteria_str):
 
 def apply_criteria(criteria, tm):
     """Apply the given criteria object to the given template"""
-    func = best_match(criteria.text) if criteria.text else lambda x, y: False
+    text = criteria.text
+    if text and isinstance(text, str):
+        text = text.decode(tm.get_template().encoding or 'utf-8')
+    func = best_match(text) if text else lambda x, y: False
     sel = tm.select(func)
     if criteria.number is not None:
         if criteria.number < len(sel):

From acbc8172b8d4aa169481d05b4109068ffb9e8f23 Mon Sep 17 00:00:00 2001
From: xyb <xieyanbo@gmail.com>
Date: Sun, 29 Sep 2013 16:37:24 +0800
Subject: [PATCH 3/3] add unittest

---
 scrapely/tests/test_template.py | 38 +++++++++++++++++++++++++------
 scrapely/tests/test_tool.py     | 40 +++++++++++++++++++++++++++++++++
 scrapely/tool.py                |  2 ++
 3 files changed, 73 insertions(+), 7 deletions(-)
 create mode 100644 scrapely/tests/test_tool.py

diff --git a/scrapely/tests/test_template.py b/scrapely/tests/test_template.py
index bce771d..2937562 100644
--- a/scrapely/tests/test_template.py
+++ b/scrapely/tests/test_template.py
@@ -1,3 +1,5 @@
+# encoding: utf8
+
 from unittest import TestCase
 
 from scrapely.htmlpage import HtmlPage
@@ -5,7 +7,19 @@
     FragmentAlreadyAnnotated, best_match
 from scrapely.extraction import InstanceBasedLearningExtractor
 
-class TemplateMakerTest(TestCase):
+
+class BaseTestCase(TestCase):
+    PAGE = HtmlPage("http://www.example.com", body=u'')
+
+    def _matches(self, text):
+        bm = best_match(text)
+        matches = [(bm(f, self.PAGE), f) for f in self.PAGE.parsed_body]
+        matches = [x for x in matches if x[0]]
+        matches.sort(reverse=True)
+        return [self.PAGE.fragment_data(x[1]) for x in matches]
+
+
+class TemplateMakerTest(BaseTestCase):
 
     PAGE = HtmlPage("http://www.example.com", body=u"""
     <html>
@@ -72,9 +86,19 @@ def test_best_match(self):
         self.assertEquals(self._matches('text to annotate'),
             ['Some text to annotate here', 'Another text to annotate there'])
 
-    def _matches(self, text):
-        bm = best_match(text)
-        matches = [(bm(f, self.PAGE), f) for f in self.PAGE.parsed_body]
-        matches = [x for x in matches if x[0]]
-        matches.sort(reverse=True)
-        return [self.PAGE.fragment_data(x[1]) for x in matches]
+
+class TemplateMakerCJKTest(BaseTestCase):
+
+    PAGE = HtmlPage("http://www.example.com", body=u"""
+    <html>
+      <body>
+        <h1>标题</h1>
+        <p>段落</p>
+        <h2>另一个标题</h2>
+        <p>另一个段落</p>
+      </body>
+    </html>
+    """)
+
+    def test_best_match(self):
+        self.assertEquals(self._matches(u'标题'), [u'标题', u'另一个标题'])
diff --git a/scrapely/tests/test_tool.py b/scrapely/tests/test_tool.py
new file mode 100644
index 0000000..0e3f305
--- /dev/null
+++ b/scrapely/tests/test_tool.py
@@ -0,0 +1,40 @@
+# encoding: utf8
+
+from unittest import TestCase
+
+from scrapely.htmlpage import HtmlPage
+from scrapely.template import TemplateMaker
+from scrapely.tool import parse_criteria, apply_criteria, readable_repr
+
+
+class ToolCJKTestCase(TestCase):
+
+    PAGE = HtmlPage("http://www.example.com", body=u"""
+    <html>
+      <body>
+        <h1>标题</h1>
+        <p>段落</p>
+        <h2>另一个标题</h2>
+        <p>另一个段落</p>
+      </body>
+    </html>
+    """)
+
+    def test_apply_criteria_should_support_cjk_chars(self):
+        criteria = parse_criteria('标题')
+        tm = TemplateMaker(self.PAGE)
+
+        selection = apply_criteria(criteria, tm)
+
+        self.assertEqual(selection, [6, 14])
+        self.assertEqual(tm.selected_data(6), u'<h1>标题</h1>')
+        self.assertEqual(tm.selected_data(14), u'<h2>另一个标题</h2>')
+
+
+class ReadableReprTextCase(TestCase):
+
+    def test_readable_repr(self):
+        cjk = u'cjk\t中日韩\n\\u535a'
+        readable = u"u'cjk\\t中日韩\\n\\\\u535a'"
+
+        self.assertEqual(readable_repr(cjk), readable)
diff --git a/scrapely/tool.py b/scrapely/tool.py
index 5b6bb02..c84c7df 100644
--- a/scrapely/tool.py
+++ b/scrapely/tool.py
@@ -11,6 +11,8 @@
 
 def readable_repr(obj):
     '''Return printing-friendly unicode repr string
+
+    Make CJK characters still readable like ASCII if you print it.
     '''
     def replace_unicode_char(repr_char):
         return unichr(int(str(repr_char.group())[2:], base=16))