Skip to content

Commit 8044a17

Browse files
rjrudinrjrudin
authored
Merge pull request #1480 from marklogic/feature/off-by-one-job-failures
Fixing stop-transform job tests
2 parents f99afbf + 1e6c8f2 commit 8044a17

File tree

4 files changed

+97
-182
lines changed

4 files changed

+97
-182
lines changed

‎Jenkinsfile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,11 @@ def runtests(String type, String version){
4040
sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml
4141
cd $WORKSPACE/java-client-api/ml-development-tools/build/test-results/test/
4242
sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml
43-
cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/test/
43+
cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFragileTests/
44+
sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml
45+
cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests/
46+
sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml
47+
cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runSlowFunctionalTests/
4448
sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml
4549
'''
4650
}

‎marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/ApplyTransformTest.java‎

Lines changed: 68 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,14 @@
1616

1717
package com.marklogic.client.datamovement.functionaltests;
1818

19-
import static org.junit.Assert.assertEquals;
20-
import static org.junit.Assert.assertFalse;
21-
import static org.junit.Assert.assertTrue;
22-
2319
import java.io.File;
2420
import java.time.Duration;
25-
import java.util.ArrayList;
26-
import java.util.Arrays;
27-
import java.util.Collections;
28-
import java.util.HashMap;
29-
import java.util.HashSet;
30-
import java.util.List;
31-
import java.util.Map;
32-
import java.util.Random;
33-
import java.util.Set;
21+
import java.util.*;
3422
import java.util.concurrent.TimeUnit;
3523
import java.util.concurrent.atomic.AtomicBoolean;
3624
import java.util.concurrent.atomic.AtomicInteger;
3725

26+
import com.marklogic.client.fastfunctest.AbstractFunctionalTest;
3827
import org.apache.commons.io.FileUtils;
3928
import org.junit.AfterClass;
4029
import org.junit.Assert;
@@ -68,18 +57,15 @@
6857
import com.marklogic.client.io.StringHandle;
6958
import com.marklogic.client.query.StructuredQueryBuilder;
7059

71-
public class ApplyTransformTest extends BasicJavaClientREST {
60+
import static org.junit.Assert.*;
61+
62+
public class ApplyTransformTest extends AbstractFunctionalTest {
7263

73-
private static String dbName = "ApplyTransform";
7464
private static DataMovementManager dmManager = null;
7565
private static final String TEST_DIR_PREFIX = "/WriteHostBatcher-testdata/";
7666

7767
private static DatabaseClient dbClient;
78-
private static String user = "admin";
79-
private static int port = 8000;
80-
private static String password = "admin";
81-
private static String server = "App-Services";
82-
68+
8369
private static JacksonHandle jacksonHandle;
8470
private static JacksonHandle jacksonHandle1;
8571
private static StringHandle stringHandle;
@@ -98,38 +84,17 @@ public class ApplyTransformTest extends BasicJavaClientREST {
9884
private static JsonNode jsonNode1;
9985
private static final String query1 = "fn:count(fn:doc())";
10086
private static String[] hostNames;
101-
private static int forestCount = 1;
87+
88+
// Number of documents to insert in each test collection
89+
private final static int DOC_COUNT = 1000;
10290

10391
/**
10492
* @throws Exception
10593
*/
10694
@BeforeClass
10795
public static void setUpBeforeClass() throws Exception {
108-
loadGradleProperties();
109-
server = getRestAppServerName();
110-
port = getRestAppServerPort();
111-
11296
hostNames = getHosts();
113-
createDB(dbName);
114-
Thread.currentThread().sleep(500L);
115-
//Ensure db has atleast one forest
116-
createForestonHost(dbName + "-" + forestCount, dbName, hostNames[0]);
117-
forestCount++;
118-
for (String forestHost : hostNames) {
119-
for(int i = 0; i < new Random().nextInt(3); i++) {
120-
createForestonHost(dbName + "-" + forestCount, dbName, forestHost);
121-
forestCount++;
122-
}
123-
Thread.currentThread().sleep(500L);
124-
}
125-
// Create App Server if needed.
126-
createRESTServerWithDB(server, port);
127-
assocRESTServer(server, dbName, port);
128-
if (IsSecurityEnabled()) {
129-
enableSecurityOnRESTServer(server, dbName);
130-
}
131-
132-
dbClient = getDatabaseClient(user, password, getConnType());
97+
dbClient = connectAsAdmin();
13398
dmManager = dbClient.newDataMovementManager();
13499

135100
// FileHandle
@@ -186,61 +151,28 @@ public static void setUpBeforeClass() throws Exception {
186151
});
187152

188153
dmManager.startJob(ihb2);
189-
outputList = new ArrayList<String>();
190-
for (int j = 0; j < 2000; j++) {
154+
outputList = new ArrayList<>();
155+
for (int j = 0; j < DOC_COUNT; j++) {
191156
String uri = "/local/json-" + j;
192157
ihb2.add(uri, meta1, fileHandle);
193158
}
194-
195-
ihb2.flushAndWait();
196-
Assert.assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 2000);
197-
198-
for (int j = 0; j < 2000; j++) {
159+
for (int j = 0; j < DOC_COUNT; j++) {
199160
String uri = "/local/string-" + j;
200161
outputList.add(uri);
201162
ihb2.add(uri, meta2, stringHandle);
202163
}
203-
ihb2.flushAndWait();
204-
205-
Assert.assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 4000);
206-
207164
ihb2.add("/local/quality", meta3, jacksonHandle);
208-
ihb2.flushAndWait();
209-
Assert.assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 4001);
210-
211165
ihb2.add("/local/nomatch", meta4, jacksonHandle1);
212-
ihb2.flushAndWait();
213-
Assert.assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 4002);
214-
215166
for (int j = 0; j < 100; j++) {
216167
String uri = "/local/snapshot-" + j;
217168
ihb2.add(uri, meta5, fileHandle);
218169
}
219-
ihb2.flushAndWait();
220-
Assert.assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 4102);
221-
222-
for (int j = 0; j < 2000; j++) {
170+
for (int j = 0; j < DOC_COUNT; j++) {
223171
String uri = "/local/skipped-" + j;
224172
ihb2.add(uri, meta6, stringHandle);
225173
}
226-
227-
ihb2.flushAndWait();
228-
Assert.assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 6102);
229-
230174
ihb2.add("/local/nonexistent-1", stringHandle);
231175
ihb2.flushAndWait();
232-
Assert.assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 6103);
233-
}
234-
235-
@AfterClass
236-
public static void tearDownAfterClass() throws Exception {
237-
associateRESTServerWithDB(server, "Documents");
238-
for (int i = 0; i < forestCount -1; i++) {
239-
detachForest(dbName, dbName + "-" + (i + 1));
240-
deleteForest(dbName + "-" + (i + 1));
241-
}
242-
243-
deleteDB(dbName);
244176
}
245177

246178
@Test
@@ -288,8 +220,8 @@ public void xQueryMasstransformReplace() throws Exception {
288220
});
289221
dmManager.startJob(resultBatcher);
290222
resultBatcher.awaitCompletion();
291-
assertEquals("document count", 2000, count.get());
292-
assertEquals("document count", 2000, success.intValue());
223+
assertEquals("document count", DOC_COUNT, count.get());
224+
assertEquals("document count", DOC_COUNT, success.intValue());
293225
assertEquals("document count", 0, skipped.intValue());
294226

295227
}
@@ -374,7 +306,7 @@ public void jsMasstransformReplace() throws Exception {
374306
});
375307
dmManager.startJob(resultBatcher);
376308
resultBatcher.awaitCompletion();
377-
assertEquals("document count", 2000, count.get());
309+
assertEquals("document count", DOC_COUNT, count.get());
378310
}
379311

380312
@Test
@@ -538,15 +470,12 @@ public void failedTransformTest() throws Exception {
538470
System.out.println(batch.getForest().getHost());
539471
urisList.addAll(Arrays.asList(batch.getItems()));
540472
});
541-
Map<String, String> properties = new HashMap<>();
542-
properties.put("locking", "strict");
543-
changeProperty(properties, "/manage/v2/databases/" + dbName + "/properties");
544473

545474
String insertQuery = "xdmp:document-insert(\"/local/failed\", "
546-
+ "<foo>This is so foo</foo>, (), \"FailTransform\", 0, xdmp:forest(\"ApplyTransform-1\") )"
475+
+ "<foo>This is so foo</foo>, (), \"FailTransform\", 0 )"
547476
+ ";xdmp:document-insert(\"/local/failed-1\", object-node {\"c\":\"v1\"}, (),"
548477
+ " \"FailTransform\","
549-
+ " 0, xdmp:forest(\"ApplyTransform-1\") )";
478+
+ " 0 )";
550479

551480
String response = dbClient.newServerEval().xquery(insertQuery).evalAs(String.class);
552481
System.out.println(response);
@@ -716,30 +645,43 @@ public void stopTransformJobTest() throws Exception {
716645
transform.put("name", "Lang");
717646
transform.put("value", "French");
718647

719-
Set<String> skippedBatch = Collections.synchronizedSet(new HashSet<String>());
720-
Set<String> successBatch = Collections.synchronizedSet(new HashSet<String>());
721-
722-
ApplyTransformListener listener = new ApplyTransformListener().withTransform(transform)
723-
.withApplyResult(ApplyResult.REPLACE).onSuccess(batch -> {
724-
successBatch.addAll(Arrays.asList(batch.getItems()));
725-
}).onSkipped(batch -> {
726-
skippedBatch.addAll(Arrays.asList(batch.getItems()));
727-
System.out.println("stopTransformJobTest : Skipped: " + batch.getItems()[0]);
728-
729-
}).onFailure((batch, throwable) -> {
730-
throwable.printStackTrace();
731-
System.out.println("stopTransformJobTest: Failed: " + batch.getItems()[0]);
648+
Set<String> skippedUris = Collections.synchronizedSet(new HashSet<>());
649+
Set<String> successUris = Collections.synchronizedSet(new HashSet<>());
650+
Set<String> failedUris = Collections.synchronizedSet(new HashSet<>());
651+
652+
List<Throwable> failures = new Vector<>();
653+
654+
ApplyTransformListener listener = new ApplyTransformListener()
655+
.withTransform(transform)
656+
.withApplyResult(ApplyResult.REPLACE).onSuccess(batch -> {
657+
successUris.addAll(Arrays.asList(batch.getItems()));
658+
}).onSkipped(batch -> {
659+
skippedUris.addAll(Arrays.asList(batch.getItems()));
660+
}).onFailure((batch, throwable) -> {
661+
failures.add(throwable);
662+
failedUris.addAll(Arrays.asList(batch.getItems()));
663+
});
664+
665+
for (Throwable failure : failures) {
666+
if (!(failure instanceof InterruptedException)) {
667+
fail("Unexpected batch failure, only expecting an InterruptedException from the batcher being interrupted: " + failure);
668+
}
669+
}
732670

733-
});
734671
QueryBatcher batcher = dmManager
735-
.newQueryBatcher(new StructuredQueryBuilder().collection("Skipped"))
736-
.onUrisReady(listener).withBatchSize(1).withThreadCount(1);
672+
.newQueryBatcher(new StructuredQueryBuilder().collection("Skipped"))
673+
.onUrisReady(listener)
674+
.withBatchSize(10)
675+
.withThreadCount(1);
676+
737677
JobTicket ticket = dmManager.startJob(batcher);
738-
Thread.currentThread().sleep(4000L);
678+
// Wait an amount of time that should result in some docs being transformed but not all
679+
Thread.currentThread().sleep(200L);
739680
dmManager.stopJob(ticket);
740681
batcher.awaitCompletion();
741-
742-
AtomicInteger count = new AtomicInteger(0);
682+
683+
// Find how many documents were transformed
684+
AtomicInteger notTransformedCount = new AtomicInteger(0);
743685
QueryBatcher resultBatcher = dmManager
744686
.newQueryBatcher(new StructuredQueryBuilder().collection("Skipped"))
745687
.withBatchSize(25).withThreadCount(5)
@@ -749,25 +691,29 @@ public void stopTransformJobTest() throws Exception {
749691
while (page.hasNext()) {
750692
DocumentRecord rec = page.next();
751693
rec.getContent(dh);
752-
if (dh.get().getElementsByTagName("foo")
753-
.item(0).getAttributes().item(0) == null) {
754-
count.incrementAndGet();
694+
if (dh.get().getElementsByTagName("foo").item(0).getAttributes().item(0) == null) {
695+
notTransformedCount.incrementAndGet();
755696
}
756-
757697
}
758-
759698
});
760699
dmManager.startJob(resultBatcher);
761700
resultBatcher.awaitCompletion();
762701

763-
System.out.println("stopTransformJobTest: Success: " + successBatch.size());
764-
System.out.println("stopTransformJobTest: Skipped: " + skippedBatch.size());
765-
System.out.println("stopTransformJobTest : count " + count);
766-
// TODO This has been failing intermittently with the sum being 1999 instead of 2000. Interestingly, the
767-
// stopJobTransform test in QueryBatcherJobReportTest sometimes fails because it's off by one as well.
768-
Assert.assertEquals(2000, successBatch.size() + skippedBatch.size() + count.get());
769-
Assert.assertEquals(2000 - count.get(), successBatch.size());
770-
702+
int successCount = successUris.size();
703+
int skippedCount = skippedUris.size();
704+
int failedCount = failedUris.size();
705+
706+
System.out.println("SUCCESS: " + successCount);
707+
System.out.println("SKIPPED: " + skippedCount);
708+
System.out.println("FAILED: " + failedCount);
709+
System.out.println("NOT TRANSFORMED: " + notTransformedCount.get());
710+
711+
assertEquals(
712+
"Unexpected count; success: " + successCount + "; skipped: " + skippedCount + "; failed: " + failedCount,
713+
DOC_COUNT, successCount + skippedCount + failedCount + notTransformedCount.get());
714+
assertEquals(
715+
"Unexpected count; success: " + successCount + "; failed: " + failedCount + "; not transformed: " + notTransformedCount.get(),
716+
DOC_COUNT - notTransformedCount.get() - failedCount, successCount);
771717
}
772718

773719
@Test

0 commit comments

Comments
 (0)