Merge branch 'release-5.5.0'

Author: itsshivaverma

CHANGELOG.md

@@ -1,16 +1,25 @@
 # CHANGELOG
 
-## 5.4.0
+## 5.5.0
 #### New Functionality
+- [#1294](https://github.com/marklogic/java-client-api/issues/1294) - New Optic features Redaction, generate views  - Server release 10.0-7 or above required
+
+#### Improvements and Bug Fixes
+- [#1230](https://github.com/marklogic/java-client-api/issues/1230) - Upgraded dependencies
+- [#1279](https://github.com/marklogic/java-client-api/issues/1279) - Size task queue for QueryBatcher based on forests, thread counts
+- [#1290](https://github.com/marklogic/java-client-api/issues/1290) - QueryBatcher fails with empty criteria
+- [#1298](https://github.com/marklogic/java-client-api/issues/1298) - Removed JCenter as repository
 
-- [#1260](https://github.com/marklogic/java-client-api/issues/1260) - CtsQueryBuilder for search is available
-- [#1280](https://github.com/marklogic/java-client-api/issues/1280) - Optic query DSL - Support for human-oriented representation of Optic query
-- [#1282](https://github.com/marklogic/java-client-api/issues/1282) - Optic enhancements - support for document matching, relevance, multiple groups
+## 5.4.0
+#### New Functionality
+- [#1260](https://github.com/marklogic/java-client-api/issues/1260) - CtsQueryBuilder for search is available - Server release 10.0-5 or above required
+- [#1280](https://github.com/marklogic/java-client-api/issues/1280) - Optic query DSL - Support for human-oriented representation of Optic query - Server release 10.0-6 or above required
+- [#1282](https://github.com/marklogic/java-client-api/issues/1282) - Optic enhancements - support for document matching, relevance, multiple groups - Server release 10.0-6 or above required
 
 #### Improvements and Bug Fixes
 - [#1256](https://github.com/marklogic/java-client-api/issues/1256) - Clear collections is not working in Java API
 - [#1268](https://github.com/marklogic/java-client-api/issues/1268) - Gradle wrapper upgrade to 6.7
-- [#1269](https://github.com/marklogic/java-client-api/issues/1269) - Increased concurrency for DMSDK QueryBatcher
+- [#1269](https://github.com/marklogic/java-client-api/issues/1269) - Increased concurrency for DMSDK QueryBatcher - Server release 10.0-6 or above required
 - [#1283](https://github.com/marklogic/java-client-api/issues/1283) - QueryBatcher fails when using path range query
 
 ## 5.3.1

NOTICE.txt

@@ -20,7 +20,6 @@ subprojects {
     repositories {
         mavenLocal()
         mavenCentral()
-        jcenter()
         maven { url "http://repo.maven.apache.org/maven2" }
     }
 

build.gradle

@@ -1,5 +1,5 @@
 group=com.marklogic
-version=5.4.0
-releaseVersion=5.4.0
+version=5.5.0
+releaseVersion=5.5.0
 describedName=MarkLogic Java Client API
 publishUrl=file:../marklogic-java/releases

gradle.properties

@@ -30,8 +30,6 @@ repositories {
 dependencies {
   implementation project (':marklogic-client-api')
   implementation group: 'org.skyscreamer', name: 'jsonassert', version: '1.5.0'
-  implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.11.0'
-  implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.11.0'
   implementation group: 'org.slf4j', name: 'slf4j-api', version:'1.7.25'
   testImplementation group: 'ch.qos.logback', name: 'logback-classic', version:'1.2.3'
   implementation group: 'commons-io', name: 'commons-io', version: '2.6'

marklogic-client-api-functionaltests/build.gradle

@@ -239,6 +239,7 @@ public void testAndWordQuery() throws Exception
       StringBuilder batchResults = new StringBuilder();
       StringBuilder forestResults = new StringBuilder();
       StringBuilder batchFailResults = new StringBuilder();
+      StringBuilder batchIllegalState = new StringBuilder();
 
       // Run a QueryBatcher on the new URIs.
       QueryBatcher queryBatcher1 = dmManager.newQueryBatcher(querydef);
@@ -276,7 +277,23 @@ public void testAndWordQuery() throws Exception
       else {
     	  fail("testAndWordQuery test failed");
       }
-    } catch (Exception e) {
+
+      try {
+        // Verify Git # 1290
+        QueryBatcher qb = dmManager.newQueryBatcher(client.newQueryManager().newStringDefinition().withCriteria("")).withBatchSize(10);
+
+        qb.onQueryFailure(throwable -> {
+          throwable.printStackTrace();
+          batchIllegalState.append(throwable.getMessage());
+        });
+        dmManager.startJob(qb);
+        qb.awaitCompletion();
+      } catch (Exception ex) {
+        batchIllegalState.append(ex.getMessage());
+        System.out.println("Exceptions buffer from empty withCriteria : " + batchIllegalState.toString());
+        assertTrue("Exception message incorrect", batchIllegalState.toString().contains("Criteria cannot be an empty string"));
+      }
+      } catch (Exception e) {
       System.out.print(e.getMessage());
     } finally {
     	clearDB();
@@ -1315,6 +1332,9 @@ public void testServerXQueryTransform() throws IOException, ParserConfigurationE
 		});
 		dmManager.startJob(queryBatcher1);
 		queryBatcher1.awaitCompletion(3, TimeUnit.MINUTES);
+        while (!queryBatcher1.isStopped()) {
+        // Do nothing. Wait for batcher to complete.
+        }
 
 		if (queryBatcher1.isStopped()) {
 		  // Verify the batch results now.

marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/StringQueryHostBatcherTest.java

@@ -476,7 +476,6 @@ public void testWriteOpsMultipleThreads() throws KeyManagementException, NoSuchA
         try {
             tempJsonFile1 = File.createTempFile("TestWriteOps1", ".json");
             BufferedWriter bwJson1 = new BufferedWriter(new FileWriter(tempJsonFile1));
-            DocumentManager docMgrJson1 = client.newJSONDocumentManager();
             dmManager = client.newDataMovementManager();
             WriteBatcher wbatcher1 = dmManager.newWriteBatcher();
 
@@ -530,15 +529,22 @@ public void testWriteOpsMultipleThreads() throws KeyManagementException, NoSuchA
             t1.join();
             t2.join();
 
-            Thread.sleep(1000);
+            wbatcher1.awaitCompletion();
+
             // Verify docs count
             QueryBatcher queryBatcherdMgr = dmManager.newQueryBatcher(
                     new StructuredQueryBuilder().collection(collectionName))
                     .withBatchSize(20)
-                    //.withThreadCount(1)
+                    .withThreadCount(1)
                     .onUrisReady((batch) -> {
                         cnt1.addAndGet(batch.getItems().length);
-                    });
+                        })
+                    .onQueryFailure(failure -> {
+                        System.out.println("query failed");
+                        failure.printStackTrace(System.out);
+                        dmManager.stopJob(failure.getBatcher());
+                        });
+
             dmManager.startJob(queryBatcherdMgr);
             queryBatcherdMgr.awaitCompletion();
             assertEquals(2 * nDocs, cnt1.get());
@@ -565,7 +571,6 @@ public void testWriteOpsMultipleThreads() throws KeyManagementException, NoSuchA
                 jsonfs1.close();
             if (jsonfs2 != null)
                 jsonfs2.close();
-            docMgrJson1 = null;
         } catch (Exception ex) {
             System.out.println("Exceptions thrown testSplitterWithMultipleThreads " + ex.getMessage());
         } finally {

marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/TestSplitters.java

@@ -78,6 +78,8 @@ public class BulkIOCallersFnTest extends BasicJavaClientREST {
 
     // Egress error endpoint ConfigName
     private static String JsonEgressErrorConfigName = "DynamicEgressServicesForJsonError";
+    // Ingest error endpoint ConfigName
+    private static String JsonIngestErrorConfigName = "DynamicIngestServicesForTextError";
 
     // Ingest and Egress endpoint ConfigName
     private static String IngestEgressSessionFieldsConfigName = "DynamicIngestEgressSessionFields";
@@ -94,6 +96,7 @@ public class BulkIOCallersFnTest extends BasicJavaClientREST {
     private static String IngestServicesXmlURI = "/dynamic/fntest/DynamicIngestServices/xml/";
     private static String IngestServicesTextURI = "/dynamic/fntest/DynamicIngestServices/text/";
     private static String IngestServicesBinURI = "/dynamic/fntest/DynamicIngestServices/bin/";
+    private static String IngestServicesJsonErrorURI = "/dynamic/fntest/DynamicIngestServicesError/json/";
     //Output URI
     private static String EgressServicesJsonURI = "/dynamic/fntest/DynamicEgressServices/json/";
     private static String EgressServicesJsonErrorURI = "/dynamic/fntest/DynamicEgressServicesError/json/";
@@ -241,6 +244,18 @@ public static void setUp() throws Exception {
         file = null;
         handle = null;
 
+        file = new File(ApiConfigDirPath + JsonIngestErrorConfigName + ".sjs");
+        handle = new FileHandle(file);
+        docMgr.write(IngestServicesJsonErrorURI + JsonIngestErrorConfigName +".sjs", metadataHandle, handle);
+        file = null;
+        handle = null;
+
+        file = new File(ApiConfigDirPath + JsonIngestErrorConfigName + ".api");
+        handle = new FileHandle(file);
+        docMgr.write(IngestServicesJsonErrorURI + JsonIngestErrorConfigName +".api", metadataHandle, handle);
+        file = null;
+        handle = null;
+
         file = new File(ApiConfigDirPath + IngestEgressSessionFieldsConfigName + ".sjs");
         handle = new FileHandle(file);
         docMgr.write(IngestEgressSessionFieldsURI + IngestEgressSessionFieldsConfigName +".sjs", metadataHandle, handle);
@@ -652,6 +667,61 @@ public void TestIngestEgressOnTextDocs() throws Exception {
         }
     }
 
+    // Use /dynamic/fntest/DynamicIngestServices/text/DynamicIngestServicesForTextError.sjs endpoint to test Text Documents ingest
+    // Verify that STOP_ALL_CALLS in client does not throw NPE. Git # 1265
+    @Test
+    public void TestIngestOnTextDocsError() throws Exception {
+        System.out.println("Running TestIngestOnTextDocsError");
+        StringBuilder errorBuf = new StringBuilder();
+        try {
+            int startBatchIdx = 0;
+            int maxDocSize = 100;
+
+            ObjectMapper om = new ObjectMapper();
+            File apiFile = new File(ApiConfigDirPath + JsonIngestErrorConfigName + ".api");
+
+            JsonNode api = om.readTree(new FileReader(apiFile));
+            JacksonHandle jhAPI = new JacksonHandle(api);
+
+            String state = "{\"next\":"+startBatchIdx+"}";
+            String work = "{\"max\":"+maxDocSize+"}";
+
+            InputCaller<String> ingressEndpt = InputCaller.on(dbclient, jhAPI, new StringHandle());
+            InputCaller.BulkInputCaller<String> inputbulkCaller = ingressEndpt.bulkCaller(ingressEndpt.newCallContext()
+                    .withEndpointConstantsAs(work.getBytes())
+                    .withEndpointStateAs(state));
+            String[] strContent = { "This is first test document",
+                    "This is second test document",
+                    "This is third test document",
+                    "This is fourth test document",
+                    "This is fifth test document"
+            };
+            InputCaller.BulkInputCaller.ErrorListener InerrorListener =
+                    (retryCount, throwable, callContext, inputHandles)
+                            -> {
+                        for(BufferableHandle h:inputHandles) {
+                            errorBuf.append(h.toString());
+                        }
+                        return IOEndpoint.BulkIOEndpointCaller.ErrorDisposition.STOP_ALL_CALLS;
+                    };
+
+            Stream<String> input = Stream.of(strContent);
+            input.forEach(inputbulkCaller::accept);
+            inputbulkCaller.setErrorListener(InerrorListener);
+            inputbulkCaller.awaitCompletion();
+            String errStr = errorBuf.toString();
+            //System.out.println("Error buffer when STOP_ALL_CALLS " + errorBuf.toString());
+            Assert.assertTrue(!errStr.contains("Exception"));
+            Assert.assertTrue(errStr.isEmpty());
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        finally {
+            System.out.println("End of TestIngestEgressOnTextDocs");
+        }
+    }
+
     // Use /dynamic/fntest/DynamicIngestServices/bin/DynamicIngestServicesForBin.sjs endpoint to test Binary Documents ingest
     @Test
     public void TestIngestEgressOnBinDocs() throws Exception {