Merge branch 'hepan-master'

Author: code4craft

webmagic-core/src/main/java/us/codecraft/webmagic/Site.java

@@ -6,6 +6,7 @@
 
 import us.codecraft.webmagic.proxy.Proxy;
 import us.codecraft.webmagic.proxy.SimpleProxyPool;
+import org.apache.http.auth.UsernamePasswordCredentials;
 import us.codecraft.webmagic.proxy.ProxyPool;
 import us.codecraft.webmagic.utils.UrlUtils;
 
@@ -53,6 +54,8 @@ public class Site {
 
     private HttpHost httpProxy;
 
+    private UsernamePasswordCredentials usernamePasswordCredentials; //代理用户名密码设置
+
     private ProxyPool httpProxyPool;
 
     private boolean useGzip = true;
@@ -486,6 +489,15 @@ public Site enableHttpProxyPool() {
         return this;
     }
 
+    public UsernamePasswordCredentials getUsernamePasswordCredentials() {
+        return usernamePasswordCredentials;
+    }
+
+    public Site setUsernamePasswordCredentials(UsernamePasswordCredentials usernamePasswordCredentials) {
+        this.usernamePasswordCredentials = usernamePasswordCredentials;
+        return this;
+    }
+
     public ProxyPool getHttpProxyPool() {
         return httpProxyPool;
     }

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java

@@ -81,6 +81,15 @@ public void process(
                 }
             });
         }
+
+        if(site!=null&&site.getHttpProxy()!=null&&site.getUsernamePasswordCredentials()!=null){
+            CredentialsProvider credsProvider = new BasicCredentialsProvider();
+            credsProvider.setCredentials(
+                    new AuthScope(site.getHttpProxy()),//可以访问的范围
+                    site.getUsernamePasswordCredentials());//用户名和密码
+            httpClientBuilder.setDefaultCredentialsProvider(credsProvider);
+        }
+
         SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
         httpClientBuilder.setDefaultSocketConfig(socketConfig);
         if (site != null) {