代理增加用户名密码认证

Author: hepan

webmagic-core/src/main/java/us/codecraft/webmagic/Site.java

@@ -4,6 +4,7 @@
 import com.google.common.collect.Table;
 import org.apache.http.HttpHost;
 
+import org.apache.http.auth.UsernamePasswordCredentials;
 import us.codecraft.webmagic.proxy.ProxyPool;
 import us.codecraft.webmagic.utils.UrlUtils;
 
@@ -51,6 +52,8 @@ public class Site {
 
     private HttpHost httpProxy;
 
+    private UsernamePasswordCredentials usernamePasswordCredentials; //代理用户名密码设置
+
     private ProxyPool httpProxyPool;
 
     private boolean useGzip = true;
@@ -479,6 +482,15 @@ public Site enableHttpProxyPool() {
         return this;
     }
 
+    public UsernamePasswordCredentials getUsernamePasswordCredentials() {
+        return usernamePasswordCredentials;
+    }
+
+    public Site setUsernamePasswordCredentials(UsernamePasswordCredentials usernamePasswordCredentials) {
+        this.usernamePasswordCredentials = usernamePasswordCredentials;
+        return this;
+    }
+
     public ProxyPool getHttpProxyPool() {
         return httpProxyPool;
     }

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java

@@ -3,7 +3,10 @@
 import org.apache.http.HttpException;
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CookieStore;
+import org.apache.http.client.CredentialsProvider;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.config.SocketConfig;
@@ -65,6 +68,15 @@ public void process(
                 }
             });
         }
+
+        if(site!=null&&site.getHttpProxy()!=null&&site.getUsernamePasswordCredentials()!=null){
+            CredentialsProvider credsProvider = new BasicCredentialsProvider();
+            credsProvider.setCredentials(
+                    new AuthScope(site.getHttpProxy()),//可以访问的范围
+                    site.getUsernamePasswordCredentials());//用户名和密码
+            httpClientBuilder.setDefaultCredentialsProvider(credsProvider);
+        }
+
         SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
         httpClientBuilder.setDefaultSocketConfig(socketConfig);
         if (site != null) {