Skip to content

Commit 758aa18

Browse files
WeiXinChanWeiXinChan
authored
Merge pull request #180 from oceanbase/fix-case-stability
fix case stability
2 parents c7ab71c + 8a4cf37 commit 758aa18

13 files changed

+348
-290
lines changed

‎src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java‎

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,7 @@
2121
import com.alipay.oceanbase.rpc.constant.Constants;
2222
import com.google.common.base.Objects;
2323
import org.apache.hadoop.classification.InterfaceAudience;
24-
import org.apache.hadoop.conf.Configuration;
25-
import org.apache.hadoop.hbase.client.ConnectionConfiguration;
24+
2625

2726
import java.io.IOException;
2827
import java.util.Map;
@@ -38,7 +37,7 @@
3837
public class ObTableClientManager {
3938

4039
public static final ConcurrentHashMap<ObTableClientKey, ReentrantLock> OB_TABLE_CLIENT_LOCK = new ConcurrentHashMap<ObTableClientKey, ReentrantLock>();
41-
public static final Map<ObTableClientKey, ObTableClient> OB_TABLE_CLIENT_INSTANCE = new ConcurrentHashMap<ObTableClientKey, ObTableClient>();
40+
public static final Map<ObTableClientKey, ObTableClient> OB_TABLE_CLIENT_INSTANCE = new ConcurrentHashMap<>();
4241

4342
public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration connectionConfig)
4443
throws IllegalArgumentException,
@@ -58,22 +57,27 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c
5857
obTableClientKey.setDatabase(connectionConfig.getDatabase());
5958
} else {
6059
checkArgument(isNotBlank(connectionConfig.getParamUrl()), HBASE_OCEANBASE_PARAM_URL
61-
+ " is blank");
62-
obTableClientKey = new ObTableClientKey();
63-
String paramUrl = connectionConfig.getParamUrl();
64-
if (!paramUrl.contains("database")) {
65-
paramUrl += "&database=default";
66-
}
67-
obTableClientKey.setParamUrl(paramUrl);
68-
obTableClientKey.setSysUserName(connectionConfig.getSysUsername());
69-
if (connectionConfig.getSysPassword() == null) {
70-
obTableClientKey.setSysPassword(Constants.EMPTY_STRING);
71-
} else {
72-
obTableClientKey.setSysPassword(connectionConfig.getSysPassword());
73-
}
60+
+ " is blank");
61+
obTableClientKey = generateObTableClientKey(connectionConfig);
62+
}
63+
return getOrCreateObTableClient(obTableClientKey, connectionConfig.getRpcConnectTimeout());
64+
}
65+
66+
public static ObTableClientKey generateObTableClientKey(OHConnectionConfiguration connectionConfig) {
67+
ObTableClientKey obTableClientKey = new ObTableClientKey();
68+
String paramUrl = connectionConfig.getParamUrl();
69+
if (!paramUrl.contains("database")) {
70+
paramUrl += "&database=default";
71+
}
72+
obTableClientKey.setParamUrl(paramUrl);
73+
obTableClientKey.setSysUserName(connectionConfig.getSysUsername());
74+
if (connectionConfig.getSysPassword() == null) {
75+
obTableClientKey.setSysPassword(Constants.EMPTY_STRING);
76+
} else {
77+
obTableClientKey.setSysPassword(connectionConfig.getSysPassword());
7478
}
7579
checkArgument(isNotBlank(connectionConfig.getFullUsername()),
76-
HBASE_OCEANBASE_FULL_USER_NAME + " is blank");
80+
HBASE_OCEANBASE_FULL_USER_NAME + " is blank");
7781
obTableClientKey.setFullUserName(connectionConfig.getFullUsername());
7882

7983
if (connectionConfig.getPassword() == null) {
@@ -85,8 +89,8 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c
8589
for (Map.Entry<Object, Object> property : connectionConfig.getProperties().entrySet()) {
8690
obTableClientKey.getProperties().put(property.getKey(), property.getValue());
8791
}
88-
89-
return getOrCreateObTableClient(obTableClientKey, connectionConfig.getRpcConnectTimeout());
92+
93+
return obTableClientKey;
9094
}
9195

9296
public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableClientKey,

‎src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java‎

Lines changed: 121 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.alipay.oceanbase.hbase.OHTableClient;
2222
import com.alipay.oceanbase.hbase.util.ObHTableTestUtil;
2323
import com.alipay.oceanbase.hbase.util.TableTemplateManager;
24+
import com.google.common.base.Strings;
2425
import org.apache.hadoop.hbase.client.*;
2526
import org.apache.hadoop.hbase.util.Bytes;
2627
import org.junit.*;
@@ -34,6 +35,7 @@
3435

3536
import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*;
3637
import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH;
38+
import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.getConnection;
3739
import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES;
3840
import static org.junit.Assert.*;
3941

@@ -53,7 +55,7 @@ public static void before() throws Exception {
5355
@AfterClass
5456
public static void finish() throws Exception {
5557
closeDistributedExecute();
56-
dropTables(tableNames, group2tableNames);
58+
// dropTables(tableNames, group2tableNames);
5759
}
5860

5961
@Before
@@ -68,133 +70,139 @@ private static void assertNullResult(Result result) throws Exception {
6870
private static void testAppend(String tableName) throws Exception {
6971
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName));
7072
hTable.init();
73+
try {
74+
byte[] FAMILY = getColumnFamilyName(tableName).getBytes();
75+
byte[] ROW = "appendKey".getBytes();
76+
byte[] v1 = Bytes.toBytes("42");
77+
byte[] v2 = Bytes.toBytes("23");
78+
byte[][] QUALIFIERS = new byte[][]{Bytes.toBytes("b"), Bytes.toBytes("a"),
79+
Bytes.toBytes("c")};
80+
Append a = new Append(ROW);
81+
a.add(FAMILY, QUALIFIERS[0], v1);
82+
a.add(FAMILY, QUALIFIERS[1], v2);
83+
a.setReturnResults(false);
84+
assertNullResult(hTable.append(a));
7185

72-
byte[] FAMILY = getColumnFamilyName(tableName).getBytes();
73-
byte[] ROW = "appendKey".getBytes();
74-
byte[] v1 = Bytes.toBytes("42");
75-
byte[] v2 = Bytes.toBytes("23");
76-
byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"),
77-
Bytes.toBytes("c") };
78-
Append a = new Append(ROW);
79-
a.add(FAMILY, QUALIFIERS[0], v1);
80-
a.add(FAMILY, QUALIFIERS[1], v2);
81-
a.setReturnResults(false);
82-
assertNullResult(hTable.append(a));
83-
84-
a = new Append(ROW);
85-
a.add(FAMILY, QUALIFIERS[0], v2);
86-
a.add(FAMILY, QUALIFIERS[1], v1);
87-
a.add(FAMILY, QUALIFIERS[2], v2);
88-
Result r = hTable.append(a);
89-
assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0])));
90-
assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1])));
91-
// QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
92-
assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
93-
assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(),
94-
r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp());
95-
96-
Get get = new Get(ROW);
97-
get.setMaxVersions(10);
98-
get.addFamily(FAMILY);
99-
Result result = hTable.get(get);
100-
assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[0]).size());
101-
assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[1]).size());
102-
assertEquals(1, result.getColumnCells(FAMILY, QUALIFIERS[2]).size());
103-
assertEquals(
104-
0,
105-
Bytes.compareTo(Bytes.add(v1, v2), result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0)
106-
.getValue()));
107-
assertEquals(0,
108-
Bytes.compareTo(v2, result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0).getValue()));
86+
a = new Append(ROW);
87+
a.add(FAMILY, QUALIFIERS[0], v2);
88+
a.add(FAMILY, QUALIFIERS[1], v1);
89+
a.add(FAMILY, QUALIFIERS[2], v2);
90+
Result r = hTable.append(a);
91+
assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0])));
92+
assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1])));
93+
// QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
94+
assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
95+
assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(),
96+
r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp());
10997

110-
hTable.close();
98+
Get get = new Get(ROW);
99+
get.setMaxVersions(10);
100+
get.addFamily(FAMILY);
101+
Result result = hTable.get(get);
102+
assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[0]).size());
103+
assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[1]).size());
104+
assertEquals(1, result.getColumnCells(FAMILY, QUALIFIERS[2]).size());
105+
assertEquals(
106+
0,
107+
Bytes.compareTo(Bytes.add(v1, v2), result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0)
108+
.getValue()));
109+
assertEquals(0,
110+
Bytes.compareTo(v2, result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0).getValue()));
111+
} finally {
112+
hTable.close();
113+
}
111114
}
112115

113116
private static void testAppendBorder(String tableName) throws Exception {
114117
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName));
115118
hTable.init();
116-
117-
byte[] FAMILY = getColumnFamilyName(tableName).getBytes();
118-
byte[] ROW = "appendKey".getBytes();
119-
byte[] v1 = Bytes.toBytes("ab");
120-
byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"),
121-
Bytes.toBytes("c") };
122-
Put put = new Put(ROW);
123-
put.addColumn(FAMILY, QUALIFIERS[1], v1);
124-
hTable.put(put);
125-
Append a = new Append(ROW);
126-
a.add(FAMILY, QUALIFIERS[1], v1);
127-
a.add(FAMILY, QUALIFIERS[2], "".getBytes());
128-
hTable.append(a);
129-
Get get = new Get(ROW);
130-
get.setMaxVersions(10);
131-
get.addFamily(FAMILY);
132-
Result result = hTable.get(get);
133-
assertEquals(3, result.size());
134-
135-
a = new Append(ROW);
136-
a.add(FAMILY, QUALIFIERS[2], v1);
137-
a.add(FAMILY, QUALIFIERS[2], "".getBytes());
138-
hTable.append(a);
139-
get = new Get(ROW);
140-
get.setMaxVersions(10);
141-
get.addFamily(FAMILY);
142-
result = hTable.get(get);
143-
assertEquals(4, result.size());
144-
145-
byte[] randomBytes = new byte[1025];
146-
Random random = new Random();
147-
random.nextBytes(randomBytes);
148-
a = new Append(ROW);
149-
a.add(FAMILY, QUALIFIERS[2], randomBytes);
150119
try {
120+
byte[] FAMILY = getColumnFamilyName(tableName).getBytes();
121+
byte[] ROW = "appendKey".getBytes();
122+
byte[] v1 = Bytes.toBytes("ab");
123+
byte[][] QUALIFIERS = new byte[][]{Bytes.toBytes("b"), Bytes.toBytes("a"),
124+
Bytes.toBytes("c")};
125+
Put put = new Put(ROW);
126+
put.addColumn(FAMILY, QUALIFIERS[1], v1);
127+
hTable.put(put);
128+
Append a = new Append(ROW);
129+
a.add(FAMILY, QUALIFIERS[1], v1);
130+
a.add(FAMILY, QUALIFIERS[2], "".getBytes());
151131
hTable.append(a);
152-
fail("unexpect error, too long data should fail");
153-
} catch (IOException e) {
154-
assertTrue(e.getCause().getMessage().contains("Data too long for column 'V'"));
155-
}
132+
Get get = new Get(ROW);
133+
get.setMaxVersions(10);
134+
get.addFamily(FAMILY);
135+
Result result = hTable.get(get);
136+
assertEquals(3, result.size());
156137

157-
hTable.close();
138+
a = new Append(ROW);
139+
a.add(FAMILY, QUALIFIERS[2], v1);
140+
a.add(FAMILY, QUALIFIERS[2], "".getBytes());
141+
hTable.append(a);
142+
get = new Get(ROW);
143+
get.setMaxVersions(10);
144+
get.addFamily(FAMILY);
145+
result = hTable.get(get);
146+
assertEquals(4, result.size());
147+
148+
byte[] randomBytes = new byte[1025];
149+
Random random = new Random();
150+
random.nextBytes(randomBytes);
151+
a = new Append(ROW);
152+
a.add(FAMILY, QUALIFIERS[2], randomBytes);
153+
try {
154+
hTable.append(a);
155+
fail("unexpect error, too long data should fail");
156+
} catch (IOException e) {
157+
assertTrue(e.getCause().getMessage().contains("Data too long for column 'V'"));
158+
}
159+
} finally {
160+
hTable.close();
161+
}
158162

159163
}
160164

161165
private static void testAppendCon(String tableName) throws Exception {
162166
OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName));
163-
hTable.init();
164-
byte[] FAMILY = getColumnFamilyName(tableName).getBytes();
165-
String column = "appColumn";
166-
byte[] ROW = "appendKey".getBytes();
167-
byte[] v = "a".getBytes();
168-
byte[] expect = "a".getBytes();
169-
ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100,100);
170-
AtomicInteger atomicInteger = new AtomicInteger(0);
171-
CountDownLatch countDownLatch = new CountDownLatch(100);
172-
for (int i = 0; i < 100; i++) {
173-
Append append = new Append(ROW);
174-
append.add(FAMILY, column.getBytes(), v);
175-
threadPoolExecutor.submit(() -> {
176-
try {
177-
hTable.append(append);
178-
atomicInteger.incrementAndGet();
179-
} catch (Exception e) {
180-
if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT") && !e.getCause().getMessage().contains("OB_TIMEOUT")) {
181-
throw new RuntimeException(e);
167+
try {
168+
hTable.init();
169+
byte[] FAMILY = getColumnFamilyName(tableName).getBytes();
170+
String column = "appColumn";
171+
byte[] ROW = "appendKey".getBytes();
172+
byte[] v = "a".getBytes();
173+
ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100, 100);
174+
AtomicInteger atomicInteger = new AtomicInteger(0);
175+
CountDownLatch countDownLatch = new CountDownLatch(100);
176+
177+
for (int i = 0; i < 100; i++) {
178+
Append append = new Append(ROW);
179+
append.add(FAMILY, column.getBytes(), v);
180+
threadPoolExecutor.submit(() -> {
181+
try {
182+
hTable.append(append);
183+
} catch (Exception e) {
184+
if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT")
185+
&& !e.getCause().getMessage().contains("OB_TIMEOUT")) {
186+
throw new RuntimeException(e);
187+
}
188+
} finally {
189+
atomicInteger.incrementAndGet();
190+
countDownLatch.countDown();
182191
}
183-
} finally {
184-
countDownLatch.countDown();
185-
}
186-
});
187-
}
188-
countDownLatch.await(100000, TimeUnit.MILLISECONDS);
189-
for (int i = 0; i < atomicInteger.get() - 1; i++) {
190-
expect = Bytes.add(expect, v);
192+
});
193+
}
194+
threadPoolExecutor.shutdown();
195+
countDownLatch.await(100000, TimeUnit.MILLISECONDS);
196+
final byte[] expect = Strings.repeat("a", atomicInteger.get()).getBytes();
197+
System.out.println("atomicInteger: " + atomicInteger.get());
198+
Get get = new Get(ROW);
199+
get.setMaxVersions(1);
200+
get.addColumn(FAMILY, column.getBytes());
201+
Result result = hTable.get(get);
202+
ObHTableTestUtil.Assert(tableName, ()-> assertTrue(0 <= Bytes.compareTo(expect, result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue())));
203+
} finally {
204+
hTable.close();
191205
}
192-
Get get = new Get(ROW);
193-
get.setMaxVersions(1);
194-
get.addColumn(FAMILY, column.getBytes());
195-
Result result = hTable.get(get);
196-
assertEquals(0, Bytes.compareTo(expect, result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue()));
197-
hTable.close();
198206
}
199207

200208
private static void testAppendMultiCF(Map.Entry<String, List<String>> entry) throws Exception {
@@ -260,8 +268,8 @@ public void testAppendMultiCF() throws Throwable {
260268

261269
@Test
262270
public void testAppendSeires() throws Throwable {
263-
createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, group2tableNames, true);
271+
createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true);
264272
FOR_EACH(series_tables, OHTableSecondaryPartAppendTest::testAppendSeires);
265-
dropTables(series_tables, group2tableNames);
273+
dropTables(series_tables, null);
266274
}
267275
}

‎src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public class OHTableSecondaryPartBatchGetTest {
4242
@BeforeClass
4343
public static void before() throws Exception {
4444
openDistributedExecute();
45-
for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_AND_SERIES_TABLES) {
45+
for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) {
4646
createTables(type, tableNames, group2tableNames, true);
4747
}
4848
}
@@ -94,8 +94,9 @@ public static void testBatchGetImpl(String tableName) throws Exception {
9494
System.out.println("Column: " + Q + ", Value: " + V);
9595
}
9696
}
97+
hTable.close();
9798
}
98-
99+
99100
@Test
100101
public void testBatchGet() throws Throwable {
101102
FOR_EACH(tableNames, OHTableSecondaryPartBatchGetTest::testBatchGetImpl);

0 commit comments

Comments
 (0)