Expose Page to onSuccess & onError.

Author: sutra

webmagic-core/src/main/java/us/codecraft/webmagic/Page.java

@@ -56,8 +56,27 @@ public class Page {
     public Page() {
     }
 
-    public static Page fail(){
+    /**
+     * Returns a {@link Page} with {@link #downloadSuccess} is {@code false}.
+     *
+     * @return the page.
+     * @deprecated Use {@link #fail(Request)} instead.
+     */
+    @Deprecated
+    public static Page fail() {
+        return fail(null);
+    }
+
+    /**
+     * Returns a {@link Page} with {@link #downloadSuccess} is {@code false},
+     * and {@link #request} is specified.
+     *
+     * @return the page.
+     * @since 0.10.0
+     */
+    public static Page fail(Request request){
         Page page = new Page();
+        page.setRequest(request);
         page.setDownloadSuccess(false);
         return page;
     }

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java

@@ -36,26 +36,62 @@ public Html download(String url, String charset) {
         return (Html) page.getHtml();
     }
 
+    /**
+     * @param request the {@link Request}.
+     * @deprecated Use {@link #onSuccess(Page, Task)} instead.
+     */
     @Deprecated
     protected void onSuccess(Request request) {
     }
 
     /**
+     * @param request the {@link Request}.
+     * @param task the {@link Task}.
      * @since 0.7.6
+     * @deprecated Use {@link #onSuccess(Page, Task)} instead.
      */
+    @Deprecated
     protected void onSuccess(Request request, Task task) {
         this.onSuccess(request);
     }
 
+    /**
+     * @param page the {@link Page}.
+     * @param task the {@link Task}.
+     * @since 0.10.0
+     */
+    protected void onSuccess(Page page, Task task) {
+        this.onSuccess(page.getRequest(), task);
+    }
+
+    /**
+     * @param request the {@link Request}.
+     * @deprecated Use {@link #onError(Page, Task, Throwable)} instead.
+     */
     @Deprecated
     protected void onError(Request request) {
     }
 
     /**
+     * @param request the {@link Request}.
+     * @param task the {@link Task}.
+     * @param e the exception.
      * @since 0.7.6
+     * @deprecated Use {@link #onError(Page, Task, Throwable)} instead.
      */
+    @Deprecated
     protected void onError(Request request, Task task, Throwable e) {
         this.onError(request);
     }
 
+    /**
+     * @param page the {@link Page}.
+     * @param task the {@link Task}.
+     * @param e the exception.
+     * @since 0.10.0
+     */
+    protected void onError(Page page, Task task, Throwable e) {
+        this.onError(page.getRequest(), task, e);
+    }
+
 }

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -79,19 +79,19 @@ public Page download(Request request, Task task) {
         CloseableHttpClient httpClient = getHttpClient(task.getSite());
         Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(request, task) : null;
         HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy);
-        Page page = Page.fail();
+        Page page = Page.fail(request);
         try {
             httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
             page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
 
-            onSuccess(request, task);
-            logger.info("download page success {}", request.getUrl());
+            onSuccess(page, task);
+            logger.info("downloading page success {}", request.getUrl());
 
             return page;
         } catch (IOException e) {
 
-            onError(request, task, e);
-            logger.info("download page error {}", request.getUrl(), e);
+            onError(page, task, e);
+            logger.info("download page {} error", request.getUrl(), e);
 
             return page;
         } finally {

webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java

@@ -88,7 +88,7 @@ public Page download(Request request, Task task) {
             logger.info("downloading page: " + request.getUrl());
         }
 
-        Page page = Page.fail();
+        Page page = Page.fail(request);
         try {
             String content = getPage(request);
             if (!content.contains("HTTP request failed")) {
@@ -98,9 +98,9 @@ public Page download(Request request, Task task) {
                 page.setRequest(request);
                 page.setStatusCode(200);
             }
-            onSuccess(request, task);
+            onSuccess(page, task);
         } catch (Exception e) {
-            onError(request, task, e);
+            onError(page, task, e);
             logger.warn("download page {} error", request.getUrl(), e);
         }
         return page;

webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java

@@ -74,7 +74,7 @@ public SeleniumDownloader setSleepTime(int sleepTime) {
     public Page download(Request request, Task task) {
         checkInit();
         WebDriver webDriver = null;
-        Page page = Page.fail();
+        Page page = Page.fail(request);
         try {
             webDriver = webDriverPool.get();
 
@@ -111,10 +111,10 @@ public Page download(Request request, Task task) {
             page.setHtml(new Html(content, request.getUrl()));
             page.setUrl(new PlainText(request.getUrl()));
             page.setRequest(request);
-            onSuccess(request, task);
+            onSuccess(page, task);
         } catch (Exception e) {
             logger.warn("download page {} error", request.getUrl(), e);
-            onError(request, task, e);
+            onError(page, task, e);
         } finally {
             if (webDriver != null) {
                 webDriverPool.returnToPool(webDriver);