Skip to content

Commit 0e01550

Browse files
sutrasutra
committed
Upgrade dependencies, including the jedis from 2.9.3 to 3.4.1.
1 parent 0d73f08 commit 0e01550

File tree

4 files changed

+64
-106
lines changed

4 files changed

+64
-106
lines changed

‎pom.xml‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -73,17 +73,17 @@
7373
<dependency>
7474
<groupId>org.apache.httpcomponents</groupId>
7575
<artifactId>httpcore</artifactId>
76-
<version>4.4.13</version>
76+
<version>4.4.14</version>
7777
</dependency>
7878
<dependency>
7979
<groupId>com.google.guava</groupId>
8080
<artifactId>guava</artifactId>
81-
<version>30.0-android</version>
81+
<version>30.1-jre</version>
8282
</dependency>
8383
<dependency>
8484
<groupId>com.jayway.jsonpath</groupId>
8585
<artifactId>json-path</artifactId>
86-
<version>2.4.0</version>
86+
<version>2.5.0</version>
8787
</dependency>
8888
<dependency>
8989
<groupId>org.slf4j</groupId>
@@ -103,7 +103,7 @@
103103
<dependency>
104104
<groupId>com.alibaba</groupId>
105105
<artifactId>fastjson</artifactId>
106-
<version>1.2.69</version>
106+
<version>1.2.75</version>
107107
</dependency>
108108
<dependency>
109109
<groupId>com.github.dreamhead</groupId>
@@ -125,33 +125,33 @@
125125
<dependency>
126126
<groupId>org.assertj</groupId>
127127
<artifactId>assertj-core</artifactId>
128-
<version>3.16.1</version>
128+
<version>3.18.1</version>
129129
<scope>test</scope>
130130
</dependency>
131131
<dependency>
132132
<groupId>org.apache.commons</groupId>
133133
<artifactId>commons-lang3</artifactId>
134-
<version>3.10</version>
134+
<version>3.11</version>
135135
</dependency>
136136
<dependency>
137137
<groupId>commons-collections</groupId>
138138
<artifactId>commons-collections</artifactId>
139139
<version>3.2.2</version>
140140
</dependency>
141141
<dependency>
142-
<groupId>commons-io</groupId>
143-
<artifactId>commons-io</artifactId>
144-
<version>2.7</version>
145-
</dependency>
142+
<groupId>commons-io</groupId>
143+
<artifactId>commons-io</artifactId>
144+
<version>2.8.0</version>
145+
</dependency>
146146
<dependency>
147147
<groupId>org.codehaus.groovy</groupId>
148148
<artifactId>groovy-all</artifactId>
149-
<version>2.4.19</version>
149+
<version>3.0.7</version>
150150
</dependency>
151151
<dependency>
152152
<groupId>org.jruby</groupId>
153153
<artifactId>jruby</artifactId>
154-
<version>9.2.11.1</version>
154+
<version>9.2.14.0</version>
155155
</dependency>
156156
<dependency>
157157
<groupId>org.jsoup</groupId>
@@ -171,12 +171,12 @@
171171
<dependency>
172172
<groupId>net.sf.saxon</groupId>
173173
<artifactId>Saxon-HE</artifactId>
174-
<version>10.1</version>
174+
<version>10.3</version>
175175
</dependency>
176176
<dependency>
177177
<groupId>net.sourceforge.htmlcleaner</groupId>
178178
<artifactId>htmlcleaner</artifactId>
179-
<version>2.5</version>
179+
<version>2.24</version>
180180
</dependency>
181181
<dependency>
182182
<groupId>com.github.detro</groupId>
@@ -191,7 +191,7 @@
191191
<dependency>
192192
<groupId>redis.clients</groupId>
193193
<artifactId>jedis</artifactId>
194-
<version>2.9.3</version>
194+
<version>3.4.1</version>
195195
</dependency>
196196
</dependencies>
197197
</dependencyManagement>

‎webmagic-core/src/main/java/us/codecraft/webmagic/selector/LinksSelector.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
package us.codecraft.webmagic.selector;
22

3-
import org.jsoup.helper.StringUtil;
4-
import org.jsoup.nodes.Element;
5-
import org.jsoup.select.Elements;
6-
73
import java.util.ArrayList;
84
import java.util.List;
95

6+
import org.apache.commons.lang3.StringUtils;
7+
import org.jsoup.nodes.Element;
8+
import org.jsoup.select.Elements;
9+
1010
/**
1111
* Links selector based on jsoup. Use absolute url. <br>
1212
*
@@ -23,9 +23,9 @@ public String select(Element element) {
2323
@Override
2424
public List<String> selectList(Element element) {
2525
Elements elements = element.select("a");
26-
List<String> links = new ArrayList<String>(elements.size());
26+
List<String> links = new ArrayList<>(elements.size());
2727
for (Element element0 : elements) {
28-
if (!StringUtil.isBlank(element0.baseUri())) {
28+
if (StringUtils.isNotBlank(element0.baseUri())) {
2929
links.add(element0.attr("abs:href"));
3030
} else {
3131
links.add(element0.attr("href"));

‎webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisPriorityScheduler.java‎

Lines changed: 33 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,23 @@
11
package us.codecraft.webmagic.scheduler;
22

3-
import com.alibaba.fastjson.JSON;
3+
import java.util.Set;
4+
45
import org.apache.commons.codec.digest.DigestUtils;
56
import org.apache.commons.lang3.StringUtils;
7+
8+
import com.alibaba.fastjson.JSON;
9+
610
import redis.clients.jedis.Jedis;
711
import redis.clients.jedis.JedisPool;
812
import us.codecraft.webmagic.Request;
913
import us.codecraft.webmagic.Task;
1014

11-
import java.util.Set;
12-
1315
/**
1416
* the redis scheduler with priority
1517
* @author sai
1618
* Created by sai on 16-5-27.
1719
*/
18-
public class RedisPriorityScheduler extends RedisScheduler
19-
{
20+
public class RedisPriorityScheduler extends RedisScheduler {
2021

2122
private static final String ZSET_PREFIX = "zset_";
2223

@@ -37,114 +38,84 @@ public RedisPriorityScheduler(JedisPool pool) {
3738
}
3839

3940
@Override
40-
protected void pushWhenNoDuplicate(Request request, Task task)
41-
{
42-
Jedis jedis = pool.getResource();
43-
try
44-
{
45-
if(request.getPriority() > 0)
41+
protected void pushWhenNoDuplicate(Request request, Task task) {
42+
try (Jedis jedis = pool.getResource()) {
43+
if (request.getPriority() > 0) {
4644
jedis.zadd(getZsetPlusPriorityKey(task), request.getPriority(), request.getUrl());
47-
else if(request.getPriority() < 0)
45+
} else if (request.getPriority() < 0) {
4846
jedis.zadd(getZsetMinusPriorityKey(task), request.getPriority(), request.getUrl());
49-
else
47+
} else {
5048
jedis.lpush(getQueueNoPriorityKey(task), request.getUrl());
49+
}
5150

5251
setExtrasInItem(jedis, request, task);
5352
}
54-
finally
55-
{
56-
pool.returnResource(jedis);
57-
}
5853
}
5954

6055
@Override
61-
public synchronized Request poll(Task task)
62-
{
63-
Jedis jedis = pool.getResource();
64-
try
65-
{
56+
public synchronized Request poll(Task task) {
57+
try (Jedis jedis = pool.getResource()) {
6658
String url = getRequest(jedis, task);
67-
if(StringUtils.isBlank(url))
59+
if (StringUtils.isBlank(url)) {
6860
return null;
61+
}
6962
return getExtrasInItem(jedis, url, task);
7063
}
71-
finally
72-
{
73-
pool.returnResource(jedis);
74-
}
7564
}
7665

77-
private String getRequest(Jedis jedis, Task task)
78-
{
66+
private String getRequest(Jedis jedis, Task task) {
7967
String url;
8068
Set<String> urls = jedis.zrevrange(getZsetPlusPriorityKey(task), 0, 0);
81-
if(urls.isEmpty())
82-
{
69+
if (urls.isEmpty()) {
8370
url = jedis.lpop(getQueueNoPriorityKey(task));
84-
if(StringUtils.isBlank(url))
85-
{
71+
if (StringUtils.isBlank(url)) {
8672
urls = jedis.zrevrange(getZsetMinusPriorityKey(task), 0, 0);
87-
if(!urls.isEmpty())
88-
{
73+
if (!urls.isEmpty()) {
8974
url = urls.toArray(new String[0])[0];
9075
jedis.zrem(getZsetMinusPriorityKey(task), url);
9176
}
9277
}
93-
}
94-
else
95-
{
78+
} else {
9679
url = urls.toArray(new String[0])[0];
9780
jedis.zrem(getZsetPlusPriorityKey(task), url);
9881
}
9982
return url;
10083
}
10184

10285
@Override
103-
public void resetDuplicateCheck(Task task)
104-
{
105-
Jedis jedis = pool.getResource();
106-
try
107-
{
86+
public void resetDuplicateCheck(Task task) {
87+
try (Jedis jedis = pool.getResource()) {
10888
jedis.del(getSetKey(task));
10989
}
110-
finally
111-
{
112-
pool.returnResource(jedis);
113-
}
11490
}
11591

116-
private String getZsetPlusPriorityKey(Task task)
117-
{
92+
private String getZsetPlusPriorityKey(Task task) {
11893
return ZSET_PREFIX + task.getUUID() + PLUS_PRIORITY_SUFFIX;
11994
}
12095

121-
private String getQueueNoPriorityKey(Task task)
122-
{
96+
private String getQueueNoPriorityKey(Task task) {
12397
return QUEUE_PREFIX + task.getUUID() + NO_PRIORITY_SUFFIX;
12498
}
12599

126-
private String getZsetMinusPriorityKey(Task task)
127-
{
100+
private String getZsetMinusPriorityKey(Task task) {
128101
return ZSET_PREFIX + task.getUUID() + MINUS_PRIORITY_SUFFIX;
129102
}
130103

131-
private void setExtrasInItem(Jedis jedis,Request request, Task task)
132-
{
133-
if(request.getExtras() != null)
134-
{
135-
String field = DigestUtils.shaHex(request.getUrl());
104+
private void setExtrasInItem(Jedis jedis,Request request, Task task) {
105+
if (request.getExtras() != null) {
106+
String field = DigestUtils.sha1Hex(request.getUrl());
136107
String value = JSON.toJSONString(request);
137108
jedis.hset(getItemKey(task), field, value);
138109
}
139110
}
140111

141-
private Request getExtrasInItem(Jedis jedis, String url, Task task)
142-
{
112+
private Request getExtrasInItem(Jedis jedis, String url, Task task) {
143113
String key = getItemKey(task);
144-
String field = DigestUtils.shaHex(url);
114+
String field = DigestUtils.sha1Hex(url);
145115
byte[] bytes = jedis.hget(key.getBytes(), field.getBytes());
146-
if(bytes != null)
116+
if (bytes != null) {
147117
return JSON.parseObject(new String(bytes), Request.class);
118+
}
148119
return new Request(url);
149120
}
150121
}

‎webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java‎

Lines changed: 10 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package us.codecraft.webmagic.scheduler;
22

3-
import com.alibaba.fastjson.JSON;
43
import org.apache.commons.codec.digest.DigestUtils;
54
import org.apache.commons.lang3.StringUtils;
5+
6+
import com.alibaba.fastjson.JSON;
7+
68
import redis.clients.jedis.Jedis;
79
import redis.clients.jedis.JedisPool;
810
import redis.clients.jedis.JedisPoolConfig;
@@ -37,21 +39,15 @@ public RedisScheduler(JedisPool pool) {
3739

3840
@Override
3941
public void resetDuplicateCheck(Task task) {
40-
Jedis jedis = pool.getResource();
41-
try {
42+
try (Jedis jedis = pool.getResource()) {
4243
jedis.del(getSetKey(task));
43-
} finally {
44-
pool.returnResource(jedis);
4544
}
4645
}
4746

4847
@Override
4948
public boolean isDuplicate(Request request, Task task) {
50-
Jedis jedis = pool.getResource();
51-
try {
49+
try (Jedis jedis = pool.getResource()) {
5250
return jedis.sadd(getSetKey(task), request.getUrl()) == 0;
53-
} finally {
54-
pool.returnResource(jedis);
5551
}
5652

5753
}
@@ -62,7 +58,7 @@ protected void pushWhenNoDuplicate(Request request, Task task) {
6258
try {
6359
jedis.rpush(getQueueKey(task), request.getUrl());
6460
if (checkForAdditionalInfo(request)) {
65-
String field = DigestUtils.shaHex(request.getUrl());
61+
String field = DigestUtils.sha1Hex(request.getUrl());
6662
String value = JSON.toJSONString(request);
6763
jedis.hset((ITEM_PREFIX + task.getUUID()), field, value);
6864
}
@@ -100,23 +96,20 @@ private boolean checkForAdditionalInfo(Request request) {
10096

10197
@Override
10298
public synchronized Request poll(Task task) {
103-
Jedis jedis = pool.getResource();
104-
try {
99+
try (Jedis jedis = pool.getResource()) {
105100
String url = jedis.lpop(getQueueKey(task));
106101
if (url == null) {
107102
return null;
108103
}
109104
String key = ITEM_PREFIX + task.getUUID();
110-
String field = DigestUtils.shaHex(url);
105+
String field = DigestUtils.sha1Hex(url);
111106
byte[] bytes = jedis.hget(key.getBytes(), field.getBytes());
112107
if (bytes != null) {
113108
Request o = JSON.parseObject(new String(bytes), Request.class);
114109
return o;
115110
}
116111
Request request = new Request(url);
117112
return request;
118-
} finally {
119-
pool.returnResource(jedis);
120113
}
121114
}
122115

@@ -134,23 +127,17 @@ protected String getItemKey(Task task) {
134127

135128
@Override
136129
public int getLeftRequestsCount(Task task) {
137-
Jedis jedis = pool.getResource();
138-
try {
130+
try (Jedis jedis = pool.getResource()) {
139131
Long size = jedis.llen(getQueueKey(task));
140132
return size.intValue();
141-
} finally {
142-
pool.returnResource(jedis);
143133
}
144134
}
145135

146136
@Override
147137
public int getTotalRequestsCount(Task task) {
148-
Jedis jedis = pool.getResource();
149-
try {
138+
try (Jedis jedis = pool.getResource()) {
150139
Long size = jedis.scard(getSetKey(task));
151140
return size.intValue();
152-
} finally {
153-
pool.returnResource(jedis);
154141
}
155142
}
156143
}

0 commit comments

Comments
 (0)