Improvements Phase 1: Implement a simple executor service running all scans in parallel.

src/main/java/com/blackduck/integration/detect/lifecycle/AggregateOperationException.java

@@ -0,0 +1,29 @@
+package com.blackduck.integration.detect.lifecycle;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Collections;
+public class AggregateOperationException extends OperationException {
+    private final List<Exception> exceptions;
+
+    public AggregateOperationException(List<Exception> exceptions) {
+        super(new RuntimeException(exceptions.get(0)));
+        this.exceptions = new ArrayList<>(exceptions);
+    }
+
+    public List<Exception> getExceptions() {
+        return Collections.unmodifiableList(exceptions);
+    }
+
+    @Override
+    public String getMessage() {
+        StringBuilder sb = new StringBuilder("Multiple exceptions occurred:\n");
+        for (int i = 0; i < exceptions.size(); i++) {
+            sb.append(String.format("  %d. %s: %s\n", i+1,
+                    exceptions.get(i).getClass().getSimpleName(),
+                    exceptions.get(i).getMessage()));
+        }
+        return sb.toString();
+    }
+
+}

src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Queue;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ExecutorService;
@@ -911,7 +912,7 @@ public final BdioUploadResult uploadBdioIntelligentPersistent(BlackDuckRunData b
         );
     }
 
-    public final CodeLocationWaitData calculateCodeLocationWaitData(List<WaitableCodeLocationData> codeLocationCreationDatas) throws OperationException {
+    public final CodeLocationWaitData calculateCodeLocationWaitData(Queue<WaitableCodeLocationData> codeLocationCreationDatas) throws OperationException {
         return auditLog.namedInternal("Calculate Code Location Wait Data", () -> new CodeLocationWaitCalculator().calculateWaitData(codeLocationCreationDatas));
     }
 
@@ -1727,4 +1728,8 @@ private void computeMD5Base64(File file, ScassScanInitiationResult initResult) t
         }
         logger.debug("Finished MD5 file computation.");
     }
+
+    public int maxParallelProcessors() {
+        return detectConfigurationFactory.findParallelProcessors();
+    }
 }

src/main/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunner.java

@@ -8,11 +8,12 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Queue;
 
 import org.apache.http.conn.HttpHostConnectException;
 import org.jetbrains.annotations.Nullable;
@@ -50,7 +51,7 @@ public SignatureScanStepRunner(OperationRunner operationRunner, BlackDuckRunData
         this.blackDuckRunData = blackDuckRunData;
     }
 
-    public SignatureScannerCodeLocationResult runSignatureScannerOnline(String detectRunUuid, NameVersion projectNameVersion, DockerTargetData dockerTargetData, Set<String> scanIdsToWaitFor, Gson gson)
+    public SignatureScannerCodeLocationResult runSignatureScannerOnline(String detectRunUuid, NameVersion projectNameVersion, DockerTargetData dockerTargetData, Queue<String> scanIdsToWaitFor, Gson gson)
         throws DetectUserFriendlyException, OperationException, IOException {        
         ScanBatchRunner scanBatchRunner = resolveOnlineScanBatchRunner(blackDuckRunData);
 
@@ -95,12 +96,12 @@ public void runSignatureScannerOffline(String detectRunUuid, NameVersion project
         executeScan(scanBatch, scanBatchRunner, scanPaths, null, null, false, false);
     }
 
-    protected List<SignatureScannerReport> executeScan(ScanBatch scanBatch, ScanBatchRunner scanBatchRunner, List<SignatureScanPath> scanPaths, Set<String> scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel, boolean isOnline) throws OperationException, IOException {
+    protected List<SignatureScannerReport> executeScan(ScanBatch scanBatch, ScanBatchRunner scanBatchRunner, List<SignatureScanPath> scanPaths, Queue<String> scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel, boolean isOnline) throws OperationException, IOException {
         // Step 1: Run Scan CLI
         SignatureScanOuputResult scanOuputResult = operationRunner.signatureScan(scanBatch, scanBatchRunner);      
 
         // Step 2: Check results and upload BDIO
-        Set<String> failedScans = processEachScan(scanIdsToWaitFor, scanOuputResult, gson, shouldWaitAtScanLevel, scanBatch.isScassScan(), isOnline, scanBatch.isCsvArchive()); 
+        Set<String> failedScans = processEachScan(scanIdsToWaitFor, scanOuputResult, gson, shouldWaitAtScanLevel, scanBatch.isScassScan(), isOnline, scanBatch.isCsvArchive());
 
         // Step 3: Report on results
         List<SignatureScannerReport> reports = operationRunner.createSignatureScanReport(scanPaths, scanOuputResult.getScanBatchOutput().getOutputs(), failedScans);
@@ -152,22 +153,22 @@ private ScanBatchRunnerUserResult findUserProvidedScanBatchRunner(Optional<File>
         return ScanBatchRunnerUserResult.none();
     }
 
-    private Set<String> processEachScan(Set<String> scanIdsToWaitFor, SignatureScanOuputResult signatureScanOutputResult, Gson gson, boolean shouldWaitAtScanLevel, boolean scassScan, boolean isOnline, boolean isCsvArchive) throws IOException {
+    private Set<String> processEachScan(Queue<String> scanIdsToWaitFor, SignatureScanOuputResult signatureScanOutputResult, Gson gson, boolean shouldWaitAtScanLevel, boolean scassScan, boolean isOnline, boolean isCsvArchive) throws IOException {
         List<ScanCommandOutput> outputs = signatureScanOutputResult.getScanBatchOutput().getOutputs();
         Set<String> failedScans = new HashSet<>();
 
         for (ScanCommandOutput output : outputs) {
             if (output.getResult() != Result.SUCCESS) {
                 continue;
             }
-            
+
             // Check if we need to copy csv files. Only do this if the user asked for it and we are not
-            // connected to BlackDuck. If we are connected to BlackDuck the scanner is responsible for 
+            // connected to BlackDuck. If we are connected to BlackDuck the scanner is responsible for
             // sending the csv there.
             if (isCsvArchive && !isOnline) {
                 copyCsvFiles(output.getSpecificRunOutputDirectory(), operationRunner.getDirectoryManager().getCsvOutputDirectory());
             }
-            
+
             if (isOnline) {
                 File specificRunOutputDirectory = output.getSpecificRunOutputDirectory();
                 String scanOutputLocation = specificRunOutputDirectory.toString()
@@ -177,11 +178,11 @@ private Set<String> processEachScan(Set<String> scanIdsToWaitFor, SignatureScanO
                         specificRunOutputDirectory, scanOutputLocation);
             }
         }
-        
+
         return failedScans;
     }
 
-    private void processOnlineScan(Set<String> scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel,
+    private void processOnlineScan(Queue<String> scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel,
             boolean scassScan, Set<String> failedScans, ScanCommandOutput output, File specificRunOutputDirectory,
             String scanOutputLocation) throws IOException, HttpHostConnectException {
         try {
@@ -218,7 +219,7 @@ private void processOnlineScan(Set<String> scanIdsToWaitFor, Gson gson, boolean
         }
     }
 
-    private void handleNoScanStatusFile(Set<String> scanIdsToWaitFor, boolean shouldWaitAtScanLevel, boolean scassScan,
+    private void handleNoScanStatusFile(Queue<String> scanIdsToWaitFor, boolean shouldWaitAtScanLevel, boolean scassScan,
             String scanOutputLocation) {        
         if (scassScan) {
             String errorMessage = String.format("Unable to find scanOutput.json file at location: {}. Unable to upload BDIO to continue signature scan.", scanOutputLocation);

src/main/java/com/blackduck/integration/detect/lifecycle/run/step/utility/ConcurrentScanWaiter.java

@@ -0,0 +1,63 @@
+package com.blackduck.integration.detect.lifecycle.run.step.utility;
+
+import com.blackduck.integration.blackduck.api.generated.view.ProjectVersionView;
+import com.blackduck.integration.blackduck.service.model.ProjectVersionWrapper;
+import com.blackduck.integration.detect.lifecycle.OperationException;
+import com.blackduck.integration.detect.lifecycle.run.data.BlackDuckRunData;
+import com.blackduck.integration.detect.lifecycle.run.operation.OperationRunner;
+import com.blackduck.integration.rest.HttpUrl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.CompletableFuture;
+
+public class ConcurrentScanWaiter {
+
+    Logger logger = LoggerFactory.getLogger(ConcurrentScanWaiter.class);
+
+    private final OperationRunner operationRunner;
+    private final ExecutorService waitingPool;
+    private final List<CompletableFuture<Void>> activWaits = Collections.synchronizedList(new ArrayList<>());
+
+    public ConcurrentScanWaiter(ExecutorService waitingPool, OperationRunner operationRunner) {
+        this.waitingPool = waitingPool;
+        this.operationRunner = operationRunner;
+    }
+
+    public void startWaitingForScan(String scanId, BlackDuckRunData blackDuckRunData, ProjectVersionWrapper projectVersion, String threadName) {
+        if (scanId == null && !blackDuckRunData.shouldWaitAtScanLevel()) return;
+
+        CompletableFuture<Void> waitFuture = CompletableFuture.runAsync(() -> {
+            Thread.currentThread().setName(threadName + " Wait For BOM Thread");
+            try {
+
+                if (scanId == null) {
+                    logger.debug("Unexpected null scanID for project version" + projectVersion.getProjectVersionView().getVersionName()
+                            + " skipping waiting for this scan.");
+                    return;
+                }
+
+                HttpUrl bomUrl = projectVersion.getProjectVersionView().getFirstLink(ProjectVersionView.BOM_STATUS_LINK);
+                HttpUrl scanUrl = new HttpUrl(bomUrl.toString() + "/" + scanId);
+                operationRunner.waitForBomCompletion(blackDuckRunData, scanUrl);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }, waitingPool);
+
+        activWaits.add(waitFuture);
+    }
+
+    public void waitForAllScansToComplete() throws OperationException {
+        try {
+            CompletableFuture.allOf(activWaits.toArray(new CompletableFuture[0])).get();
+        } catch (ExecutionException | InterruptedException e) {
+            throw new OperationException(e);
+        }
+    }
+}

src/main/java/com/blackduck/integration/detect/lifecycle/run/step/utility/StepHelper.java

@@ -1,7 +1,10 @@
 package com.blackduck.integration.detect.lifecycle.run.step.utility;
 
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
 import java.util.function.Consumer;
+import java.util.List;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -33,6 +36,21 @@ public void runToolIfIncluded(DetectTool detectTool, String name, OperationWrapp
         }, () -> {}, (e) -> {});
     }
 
+    public void runToolIfIncluded(DetectTool detectTool, String name, OperationWrapper.OperationFunction supplier, List<CompletableFuture<Void>> scanFutures, ExecutorService executorService) {
+        scanFutures.add(CompletableFuture.runAsync(() -> {
+            Thread.currentThread().setName(name + " Thread");
+            try {
+                runToolIfIncluded(detectTool, name, () -> {
+                    supplier.execute();
+                    return true;
+                }, () -> {}, (e) -> {});
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }, executorService));
+
+    }
+
     public <T> Optional<T> runToolIfIncluded(DetectTool detectTool, String name, OperationWrapper.OperationSupplier<T> supplier) throws OperationException {
         return runToolIfIncluded(detectTool, name, supplier, () -> {}, (e) -> {});
     }

src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulator.java

@@ -1,20 +1,19 @@
 package com.blackduck.integration.detect.workflow.blackduck.codelocation;
 
-import java.util.ArrayList;
-import java.util.EnumMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 import com.blackduck.integration.blackduck.codelocation.CodeLocationBatchOutput;
 import com.blackduck.integration.blackduck.codelocation.CodeLocationCreationData;
 import com.blackduck.integration.detect.configuration.enumeration.DetectTool;
 
 public class CodeLocationAccumulator {
-    private final List<WaitableCodeLocationData> waitableCodeLocationData = new ArrayList<>();
-    private final Set<String> nonWaitableCodeLocations = new HashSet<>();
-    private final Map<DetectTool, Integer> additionalCountsByTool = new EnumMap<>(DetectTool.class);
+    private final Queue<WaitableCodeLocationData> waitableCodeLocationData = new ConcurrentLinkedQueue<>();
+    private final Queue<String> nonWaitableCodeLocations = new ConcurrentLinkedQueue<>();
+    private final Map<DetectTool, Integer> additionalCountsByTool = new ConcurrentHashMap<>();
 
     public void addWaitableCodeLocations(DetectTool detectTool, CodeLocationCreationData<? extends CodeLocationBatchOutput<?>> creationData) {
         addWaitableCodeLocations(new WaitableCodeLocationData(detectTool,
@@ -40,11 +39,11 @@ public void incrementAdditionalCounts(DetectTool tool, int count) {
         additionalCountsByTool.put(tool, additionalCountsByTool.getOrDefault(tool, 0) + count);
     }
 
-    public List<WaitableCodeLocationData> getWaitableCodeLocations() {
+    public Queue<WaitableCodeLocationData> getWaitableCodeLocations() {
         return waitableCodeLocationData;
     }
 
-    public Set<String> getNonWaitableCodeLocations() {
+    public Queue<String> getNonWaitableCodeLocations() {
         return nonWaitableCodeLocations;
     }
 

src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationWaitCalculator.java

@@ -2,13 +2,13 @@
 
 import java.util.Date;
 import java.util.HashSet;
-import java.util.List;
+import java.util.Queue;
 import java.util.Set;
 
 import com.blackduck.integration.blackduck.service.model.NotificationTaskRange;
 
 public class CodeLocationWaitCalculator {
-    public CodeLocationWaitData calculateWaitData(List<WaitableCodeLocationData> codeLocationCreationDatas) {
+    public CodeLocationWaitData calculateWaitData(Queue<WaitableCodeLocationData> codeLocationCreationDatas) {
         int expectedNotificationCount = 0;
         NotificationTaskRange notificationRange = null;
         Set<String> codeLocationNames = new HashSet<>();

src/main/java/com/blackduck/integration/detect/workflow/blackduck/integratedmatching/ScanCountsPayloadCreator.java

@@ -3,6 +3,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
 
 import org.jetbrains.annotations.NotNull;
 
@@ -13,7 +14,7 @@
 
 public class ScanCountsPayloadCreator {
 
-    public ScanCountsPayload create(List<WaitableCodeLocationData> createdCodelocations, Map<DetectTool, Integer> additionalCounts) {
+    public ScanCountsPayload create(Queue<WaitableCodeLocationData> createdCodelocations, Map<DetectTool, Integer> additionalCounts) {
         Map<DetectTool, Integer> countsByTool = collectCountsByTool(createdCodelocations, additionalCounts);
         return createPayloadFromCountsByTool(countsByTool);
     }
@@ -30,7 +31,7 @@ private ScanCountsPayload createPayloadFromCountsByTool(final Map<DetectTool, In
     }
 
     @NotNull
-    private Map<DetectTool, Integer> collectCountsByTool(final List<WaitableCodeLocationData> createdCodelocations, Map<DetectTool, Integer> additionalCounts) {
+    private Map<DetectTool, Integer> collectCountsByTool(final Queue<WaitableCodeLocationData> createdCodelocations, Map<DetectTool, Integer> additionalCounts) {
         Map<DetectTool, Integer> countsByTool = new HashMap<>(additionalCounts);
         for (WaitableCodeLocationData waitableCodeLocationData : createdCodelocations) {
             int oldCount = countsByTool.getOrDefault(waitableCodeLocationData.getDetectTool(), 0);

src/main/java/com/blackduck/integration/detect/workflow/diagnostic/DiagnosticLogger.java

@@ -53,7 +53,7 @@ private FileAppender<ILoggingEvent> addAppender(String file) {
         LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
         PatternLayoutEncoder ple = new PatternLayoutEncoder();
 
-        ple.setPattern("%date{\"yyyy-MM-dd'T'HH:mm:ss,SSSXXX\", UTC} %level [%file:%line] %msg%n");
+        ple.setPattern("%date{\"yyyy-MM-dd'T'HH:mm:ss,SSSXXX\", UTC} %level [%thread] [%file:%line] %msg%n");
         ple.setContext(lc);
         ple.start();
 

src/test/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunnerTest.java

@@ -9,6 +9,8 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.apache.http.HttpHost;
 import org.apache.http.conn.HttpHostConnectException;
@@ -56,7 +58,7 @@ public void setUp() {
     @Test
     public void testRunSignatureScannerOnlineRetriesOnHttpHostConnectException() throws Exception {
         DockerTargetData dockerTargetData = mock(DockerTargetData.class);
-        Set<String> scanIdsToWaitFor = new HashSet<>();
+        Queue<String> scanIdsToWaitFor = new ConcurrentLinkedQueue<>();
         List<SignatureScanPath> scanPaths = Collections.singletonList(mock(SignatureScanPath.class));
         ScanBatch scanBatch = mock(ScanBatch.class);
 
@@ -82,7 +84,7 @@ public void testRunSignatureScannerOnlineRetriesOnHttpHostConnectException() thr
     @Test
     public void testRunSignatureScannerOnlineExecutesOnceWithoutException() throws Exception {
         DockerTargetData dockerTargetData = mock(DockerTargetData.class);
-        Set<String> scanIdsToWaitFor = new HashSet<>();
+        Queue<String> scanIdsToWaitFor = new ConcurrentLinkedQueue<>();
         List<SignatureScanPath> scanPaths = Collections.singletonList(mock(SignatureScanPath.class));
         ScanBatch scanBatch = mock(ScanBatch.class);