diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/AggregateOperationException.java b/src/main/java/com/blackduck/integration/detect/lifecycle/AggregateOperationException.java new file mode 100644 index 0000000000..2cd1aaeb6b --- /dev/null +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/AggregateOperationException.java @@ -0,0 +1,29 @@ +package com.blackduck.integration.detect.lifecycle; + +import java.util.List; +import java.util.ArrayList; +import java.util.Collections; +public class AggregateOperationException extends OperationException { + private final List exceptions; + + public AggregateOperationException(List exceptions) { + super(new RuntimeException(exceptions.get(0))); + this.exceptions = new ArrayList<>(exceptions); + } + + public List getExceptions() { + return Collections.unmodifiableList(exceptions); + } + + @Override + public String getMessage() { + StringBuilder sb = new StringBuilder("Multiple exceptions occurred:\n"); + for (int i = 0; i < exceptions.size(); i++) { + sb.append(String.format(" %d. %s: %s\n", i+1, + exceptions.get(i).getClass().getSimpleName(), + exceptions.get(i).getMessage())); + } + return sb.toString(); + } + +} diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java index dfe34b16ec..b71ab47279 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Queue; import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutorService; @@ -911,7 +912,7 @@ public final BdioUploadResult uploadBdioIntelligentPersistent(BlackDuckRunData b ); } - public final CodeLocationWaitData calculateCodeLocationWaitData(List codeLocationCreationDatas) throws OperationException { + public final CodeLocationWaitData calculateCodeLocationWaitData(Queue codeLocationCreationDatas) throws OperationException { return auditLog.namedInternal("Calculate Code Location Wait Data", () -> new CodeLocationWaitCalculator().calculateWaitData(codeLocationCreationDatas)); } @@ -1727,4 +1728,8 @@ private void computeMD5Base64(File file, ScassScanInitiationResult initResult) t } logger.debug("Finished MD5 file computation."); } + + public int maxParallelProcessors() { + return detectConfigurationFactory.findParallelProcessors(); + } } diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IntelligentModeStepRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IntelligentModeStepRunner.java index 2133cb3ee1..2c8946f54c 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IntelligentModeStepRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IntelligentModeStepRunner.java @@ -6,10 +6,21 @@ import java.util.HashSet; import java.util.Optional; import java.util.Set; +import java.util.Queue; import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.List; +import java.util.ArrayList; +import com.blackduck.integration.detect.lifecycle.AggregateOperationException; import com.blackduck.integration.detect.lifecycle.run.data.CommonScanResult; +import com.blackduck.integration.detect.lifecycle.run.step.utility.ConcurrentScanWaiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,7 +113,12 @@ public void runOnline( DetectToolFilter detectToolFilter, DockerTargetData dockerTargetData, Set binaryTargets - ) throws OperationException { + ) throws Exception { + + ExecutorService executorService = Executors.newFixedThreadPool(operationRunner.maxParallelProcessors()); + ConcurrentScanWaiter concurrentScanWaiter = new ConcurrentScanWaiter(executorService, operationRunner); + + List> scanFutures = new ArrayList<>(); ProjectVersionWrapper projectVersion = stepHelper.runAsGroup( "Create or Locate Project", @@ -112,14 +128,21 @@ public void runOnline( logger.debug("Completed project and version actions."); logger.debug("Processing Detect Code Locations."); - - Set scanIdsToWaitFor = new HashSet<>(); + + Queue scanIdsToWaitFor = new ConcurrentLinkedQueue<>(); AtomicBoolean mustWaitAtBomSummaryLevel = new AtomicBoolean(false); CodeLocationAccumulator codeLocationAccumulator = new CodeLocationAccumulator(); if (bdioResult.isNotEmpty()) { - invokePackageManagerScanningWorkflow(projectNameVersion, blackDuckRunData, scanIdsToWaitFor, bdioResult, codeLocationAccumulator); + scanFutures.add(CompletableFuture.runAsync(() -> { + try { + Thread.currentThread().setName("Package Manager Scan Thread"); + invokePackageManagerScanningWorkflow(projectNameVersion, blackDuckRunData, scanIdsToWaitFor, bdioResult, codeLocationAccumulator, concurrentScanWaiter, projectVersion); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, executorService)); } else { logger.debug("No BDIO results to upload. Skipping."); } @@ -127,45 +150,92 @@ public void runOnline( logger.debug("Completed Detect Code Location processing."); stepHelper.runToolIfIncluded(DetectTool.SIGNATURE_SCAN, "Signature Scanner", () -> { + Queue scanIds = new ConcurrentLinkedQueue<>(); SignatureScanStepRunner signatureScanStepRunner = new SignatureScanStepRunner(operationRunner, blackDuckRunData); SignatureScannerCodeLocationResult signatureScannerCodeLocationResult = signatureScanStepRunner.runSignatureScannerOnline( - detectRunUuid, - projectNameVersion, - dockerTargetData, - scanIdsToWaitFor, - gson + detectRunUuid, + projectNameVersion, + dockerTargetData, + scanIds, + gson ); + + for (String scanId : scanIds) { + logger.debug("Waiting for signature scan id {}", scanId); + concurrentScanWaiter.startWaitingForScan(scanId, blackDuckRunData, projectVersion, "Signature Scan"); + } + codeLocationAccumulator.addWaitableCodeLocations(signatureScannerCodeLocationResult.getWaitableCodeLocationData()); codeLocationAccumulator.addNonWaitableCodeLocation(signatureScannerCodeLocationResult.getNonWaitableCodeLocationData()); - }); + }, scanFutures, executorService); + + + stepHelper.runToolIfIncluded(DetectTool.BINARY_SCAN, "Binary Scanner", () -> { + invokeBinaryScanningWorkflow(DetectTool.BINARY_SCAN, dockerTargetData, projectNameVersion, blackDuckRunData, binaryTargets, scanIdsToWaitFor, codeLocationAccumulator, mustWaitAtBomSummaryLevel, concurrentScanWaiter, projectVersion); + }, scanFutures, executorService); + - stepHelper.runToolIfIncluded(DetectTool.BINARY_SCAN, "Binary Scanner", () -> { - invokeBinaryScanningWorkflow(DetectTool.BINARY_SCAN, dockerTargetData, projectNameVersion, blackDuckRunData, binaryTargets, scanIdsToWaitFor, codeLocationAccumulator, mustWaitAtBomSummaryLevel); - }); stepHelper.runToolIfIncluded( - DetectTool.CONTAINER_SCAN, - "Container Scanner", - () -> invokeContainerScanningWorkflow(scanIdsToWaitFor, codeLocationAccumulator, blackDuckRunData, projectNameVersion) + DetectTool.CONTAINER_SCAN, + "Container Scanner", + () -> invokeContainerScanningWorkflow(scanIdsToWaitFor, codeLocationAccumulator, blackDuckRunData, projectNameVersion, concurrentScanWaiter, projectVersion), + scanFutures, + executorService ); - stepHelper.runToolIfIncludedWithCallbacks( - DetectTool.IMPACT_ANALYSIS, - "Vulnerability Impact Analysis", - () -> { - runImpactAnalysisOnline(projectNameVersion, projectVersion, codeLocationAccumulator, blackDuckRunData.getBlackDuckServicesFactory()); - mustWaitAtBomSummaryLevel.set(true); - }, - operationRunner::publishImpactSuccess, - operationRunner::publishImpactFailure - ); + + + scanFutures.add(CompletableFuture.runAsync(() -> { + try { + Thread.currentThread().setName("Impact Analysis Thread"); + stepHelper.runToolIfIncludedWithCallbacks( + DetectTool.IMPACT_ANALYSIS, + "Vulnerability Impact Analysis", + () -> { + runImpactAnalysisOnline(projectNameVersion, projectVersion, codeLocationAccumulator, blackDuckRunData.getBlackDuckServicesFactory()); + mustWaitAtBomSummaryLevel.set(true); + }, + operationRunner::publishImpactSuccess, + operationRunner::publishImpactFailure + ); + } catch (OperationException e) { + throw new RuntimeException(e); + } + }, executorService)); + + stepHelper.runToolIfIncluded(DetectTool.IAC_SCAN, "IaC Scanner", () -> { IacScanStepRunner iacScanStepRunner = new IacScanStepRunner(operationRunner); IacScanCodeLocationData iacScanCodeLocationData = iacScanStepRunner.runIacScanOnline(detectRunUuid, projectNameVersion, blackDuckRunData); codeLocationAccumulator.addNonWaitableCodeLocation(iacScanCodeLocationData.getCodeLocationNames()); mustWaitAtBomSummaryLevel.set(true); - }); + }, scanFutures, executorService); + + List exceptions = new ArrayList<>(); + + for (Future future : scanFutures) { + try { + future.get(); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException && cause.getCause() instanceof OperationException) { + exceptions.add((OperationException) cause.getCause()); + } else { + exceptions.add(new OperationException(e)); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new OperationException(e); + } + } + + if (!exceptions.isEmpty()) { + throw new AggregateOperationException(exceptions); // Multiple exceptions + } + + if (operationRunner.createBlackDuckPostOptions().isCorrelatedScanningEnabled()) { stepHelper.runAsGroup("Upload Correlated Scan Counts", OperationType.INTERNAL, () -> { @@ -180,10 +250,8 @@ public void runOnline( CodeLocationResults codeLocationResults = calculateCodeLocations(codeLocationAccumulator); if (operationRunner.createBlackDuckPostOptions().shouldWaitForResults()) { - // Waiting at the scan level is more reliable, do that if the BD server is new enough. - if (blackDuckRunData.shouldWaitAtScanLevel()) { - pollForBomScanCompletion(blackDuckRunData, projectVersion, scanIdsToWaitFor); - } + // Waiting at the scan level is more reliable, do that if the BD server is new enough.+++++++++ + concurrentScanWaiter.waitForAllScansToComplete(); // If the BD server is older, or we can't detect its version, or if we have scans that we are // not yet able to obtain the scanID for, use the original notification based waiting. @@ -201,7 +269,7 @@ public void runOnline( }); } - private void invokePackageManagerScanningWorkflow(NameVersion projectNameVersion, BlackDuckRunData blackDuckRunData, Set scanIdsToWaitFor, BdioResult bdioResult, CodeLocationAccumulator codeLocationAccumulator) throws OperationException { + private void invokePackageManagerScanningWorkflow(NameVersion projectNameVersion, BlackDuckRunData blackDuckRunData, Queue scanIdsToWaitFor, BdioResult bdioResult, CodeLocationAccumulator codeLocationAccumulator, ConcurrentScanWaiter concurrentScanWaiter, ProjectVersionWrapper projectVersion) throws OperationException { if (PackageManagerStepRunner.areScassScansPossible(blackDuckRunData.getBlackDuckServerVersion())) { PackageManagerStepRunner packageManagerScanStepRunner = new PackageManagerStepRunner(operationRunner); @@ -211,21 +279,23 @@ private void invokePackageManagerScanningWorkflow(NameVersion projectNameVersion scanId = commonScanResult.getScanId() == null ? null : commonScanResult.getScanId().toString(); if(commonScanResult.isPackageManagerScassPossible()) { scanIdsToWaitFor.add(scanId); + logger.debug("Waiting for package manager scan id {}", scanId); + concurrentScanWaiter.startWaitingForScan(scanId, blackDuckRunData, projectVersion, "Package Manager"); codeLocationAccumulator.addNonWaitableCodeLocation(commonScanResult.getCodeLocationName()); codeLocationAccumulator.incrementAdditionalCounts(DetectTool.DETECTOR, 1); return; } } - invokePreScassPackageManagerWorkflow(blackDuckRunData, bdioResult, scanIdsToWaitFor, codeLocationAccumulator, scanId); + invokePreScassPackageManagerWorkflow(blackDuckRunData, bdioResult, scanIdsToWaitFor, codeLocationAccumulator, scanId, concurrentScanWaiter, projectVersion); } else { String scanId = null; - invokePreScassPackageManagerWorkflow(blackDuckRunData, bdioResult, scanIdsToWaitFor, codeLocationAccumulator, scanId); + invokePreScassPackageManagerWorkflow(blackDuckRunData, bdioResult, scanIdsToWaitFor, codeLocationAccumulator, scanId, concurrentScanWaiter, projectVersion); } } - private void invokePreScassPackageManagerWorkflow(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, Set scanIdsToWaitFor, CodeLocationAccumulator codeLocationAccumulator, String scanId) throws OperationException { + private void invokePreScassPackageManagerWorkflow(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, Queue scanIdsToWaitFor, CodeLocationAccumulator codeLocationAccumulator, String scanId, ConcurrentScanWaiter scanWaiter, ProjectVersionWrapper projectVersion) throws OperationException { stepHelper.runAsGroup("Upload Bdio", OperationType.INTERNAL, () -> { - uploadBdio(blackDuckRunData, bdioResult, scanIdsToWaitFor, codeLocationAccumulator, operationRunner.calculateDetectTimeout(), scanId); + uploadBdio(blackDuckRunData, bdioResult, scanIdsToWaitFor, codeLocationAccumulator, operationRunner.calculateDetectTimeout(), scanId, scanWaiter, projectVersion); }); } @@ -235,9 +305,11 @@ private void invokeBinaryScanningWorkflow( NameVersion projectNameVersion, BlackDuckRunData blackDuckRunData, Set binaryTargets, - Set scanIdsToWaitFor, + Queue scanIdsToWaitFor, CodeLocationAccumulator codeLocationAccumulator, - AtomicBoolean mustWaitAtBomSummaryLevel + AtomicBoolean mustWaitAtBomSummaryLevel, + ConcurrentScanWaiter scanWaiter, + ProjectVersionWrapper projectVersion ) throws IntegrationException, OperationException { logger.debug("Invoking intelligent persistent binary scan."); @@ -251,6 +323,8 @@ private void invokeBinaryScanningWorkflow( if (scanId.isPresent()) { scanIdsToWaitFor.add(scanId.get().toString()); + logger.debug("Waiting for binary scan id {}", scanId); + scanWaiter.startWaitingForScan(scanId.get().toString(), blackDuckRunData, projectVersion, "Binary Scan"); } else { Optional> codeLocations = binaryScanStepRunner.getCodeLocations(); @@ -262,10 +336,12 @@ private void invokeBinaryScanningWorkflow( } private void invokeContainerScanningWorkflow( - Set scanIdsToWaitFor, + Queue scanIdsToWaitFor, CodeLocationAccumulator codeLocationAccumulator, BlackDuckRunData blackDuckRunData, - NameVersion projectNameVersion + NameVersion projectNameVersion, + ConcurrentScanWaiter scanWaiter, + ProjectVersionWrapper projectVersion ) throws IntegrationException, OperationException{ logger.debug("Invoking intelligent persistent container scan."); @@ -277,29 +353,19 @@ private void invokeContainerScanningWorkflow( } Optional scanId = containerScanStepRunner.invokeContainerScanningWorkflow(); - scanId.ifPresent(uuid -> scanIdsToWaitFor.add(uuid.toString())); + scanId.ifPresent(uuid -> { + scanIdsToWaitFor.add(uuid.toString()); + logger.debug("Waiting for container scan id {}", scanId); + scanWaiter.startWaitingForScan(uuid.toString(), blackDuckRunData, projectVersion, "Container Scan"); + }); Set containerScanCodeLocations = new HashSet<>(); - containerScanCodeLocations.add(containerScanStepRunner.getCodeLocationName()); - codeLocationAccumulator.addNonWaitableCodeLocation(containerScanCodeLocations); - } - - private void pollForBomScanCompletion(BlackDuckRunData blackDuckRunData, ProjectVersionWrapper projectVersion, - Set scanIdsToWaitFor) throws IntegrationException, OperationException { - HttpUrl bomToSearchFor = projectVersion.getProjectVersionView().getFirstLink(ProjectVersionView.BOM_STATUS_LINK); - - for (String scanId : scanIdsToWaitFor) { - if (scanId == null) { - logger.debug("Unexpected null scanID for project version" + projectVersion.getProjectVersionView().getVersionName() - + " skipping waiting for this scan."); - continue; - } - - HttpUrl scanToSearchFor = new HttpUrl(bomToSearchFor.toString() + "/" + scanId); - operationRunner.waitForBomCompletion(blackDuckRunData, scanToSearchFor); + if(containerScanStepRunner.getCodeLocationName() != null) { + containerScanCodeLocations.add(containerScanStepRunner.getCodeLocationName()); + codeLocationAccumulator.addNonWaitableCodeLocation(containerScanCodeLocations); } } - public void uploadBdio(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, Set scanIdsToWaitFor, CodeLocationAccumulator codeLocationAccumulator, Long timeout, String scassScanId) throws OperationException { + public void uploadBdio(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, Queue scanIdsToWaitFor, CodeLocationAccumulator codeLocationAccumulator, Long timeout, String scassScanId, ConcurrentScanWaiter concurrentScanWaiter, ProjectVersionWrapper projectVersion) throws OperationException { BdioUploadResult uploadResult = operationRunner.uploadBdioIntelligentPersistent(blackDuckRunData, bdioResult, timeout, scassScanId); Optional> codeLocationCreationData = uploadResult.getUploadOutput(); codeLocationCreationData.ifPresent(uploadBatchOutputCodeLocationCreationData -> codeLocationAccumulator.addWaitableCodeLocations( @@ -308,7 +374,10 @@ public void uploadBdio(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, )); if (uploadResult.getUploadOutput().isPresent()) { for (UploadOutput result : uploadResult.getUploadOutput().get().getOutput()) { - result.getScanId().ifPresent((scanId) -> scanIdsToWaitFor.add(scanId)); + result.getScanId().ifPresent((scanId) -> { + scanIdsToWaitFor.add(scanId); + concurrentScanWaiter.startWaitingForScan(scanId, blackDuckRunData, projectVersion, "Package Manager"); + }); } } } diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunner.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunner.java index 609271e83d..105e7ab794 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunner.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunner.java @@ -8,11 +8,12 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.Collections; +import java.util.HashSet; +import java.util.Queue; import org.apache.http.conn.HttpHostConnectException; import org.jetbrains.annotations.Nullable; @@ -50,7 +51,7 @@ public SignatureScanStepRunner(OperationRunner operationRunner, BlackDuckRunData this.blackDuckRunData = blackDuckRunData; } - public SignatureScannerCodeLocationResult runSignatureScannerOnline(String detectRunUuid, NameVersion projectNameVersion, DockerTargetData dockerTargetData, Set scanIdsToWaitFor, Gson gson) + public SignatureScannerCodeLocationResult runSignatureScannerOnline(String detectRunUuid, NameVersion projectNameVersion, DockerTargetData dockerTargetData, Queue scanIdsToWaitFor, Gson gson) throws DetectUserFriendlyException, OperationException, IOException { ScanBatchRunner scanBatchRunner = resolveOnlineScanBatchRunner(blackDuckRunData); @@ -95,12 +96,12 @@ public void runSignatureScannerOffline(String detectRunUuid, NameVersion project executeScan(scanBatch, scanBatchRunner, scanPaths, null, null, false, false); } - protected List executeScan(ScanBatch scanBatch, ScanBatchRunner scanBatchRunner, List scanPaths, Set scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel, boolean isOnline) throws OperationException, IOException { + protected List executeScan(ScanBatch scanBatch, ScanBatchRunner scanBatchRunner, List scanPaths, Queue scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel, boolean isOnline) throws OperationException, IOException { // Step 1: Run Scan CLI SignatureScanOuputResult scanOuputResult = operationRunner.signatureScan(scanBatch, scanBatchRunner); // Step 2: Check results and upload BDIO - Set failedScans = processEachScan(scanIdsToWaitFor, scanOuputResult, gson, shouldWaitAtScanLevel, scanBatch.isScassScan(), isOnline, scanBatch.isCsvArchive()); + Set failedScans = processEachScan(scanIdsToWaitFor, scanOuputResult, gson, shouldWaitAtScanLevel, scanBatch.isScassScan(), isOnline, scanBatch.isCsvArchive()); // Step 3: Report on results List reports = operationRunner.createSignatureScanReport(scanPaths, scanOuputResult.getScanBatchOutput().getOutputs(), failedScans); @@ -152,7 +153,7 @@ private ScanBatchRunnerUserResult findUserProvidedScanBatchRunner(Optional return ScanBatchRunnerUserResult.none(); } - private Set processEachScan(Set scanIdsToWaitFor, SignatureScanOuputResult signatureScanOutputResult, Gson gson, boolean shouldWaitAtScanLevel, boolean scassScan, boolean isOnline, boolean isCsvArchive) throws IOException { + private Set processEachScan(Queue scanIdsToWaitFor, SignatureScanOuputResult signatureScanOutputResult, Gson gson, boolean shouldWaitAtScanLevel, boolean scassScan, boolean isOnline, boolean isCsvArchive) throws IOException { List outputs = signatureScanOutputResult.getScanBatchOutput().getOutputs(); Set failedScans = new HashSet<>(); @@ -160,14 +161,14 @@ private Set processEachScan(Set scanIdsToWaitFor, SignatureScanO if (output.getResult() != Result.SUCCESS) { continue; } - + // Check if we need to copy csv files. Only do this if the user asked for it and we are not - // connected to BlackDuck. If we are connected to BlackDuck the scanner is responsible for + // connected to BlackDuck. If we are connected to BlackDuck the scanner is responsible for // sending the csv there. if (isCsvArchive && !isOnline) { copyCsvFiles(output.getSpecificRunOutputDirectory(), operationRunner.getDirectoryManager().getCsvOutputDirectory()); } - + if (isOnline) { File specificRunOutputDirectory = output.getSpecificRunOutputDirectory(); String scanOutputLocation = specificRunOutputDirectory.toString() @@ -177,11 +178,11 @@ private Set processEachScan(Set scanIdsToWaitFor, SignatureScanO specificRunOutputDirectory, scanOutputLocation); } } - + return failedScans; } - private void processOnlineScan(Set scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel, + private void processOnlineScan(Queue scanIdsToWaitFor, Gson gson, boolean shouldWaitAtScanLevel, boolean scassScan, Set failedScans, ScanCommandOutput output, File specificRunOutputDirectory, String scanOutputLocation) throws IOException, HttpHostConnectException { try { @@ -218,7 +219,7 @@ private void processOnlineScan(Set scanIdsToWaitFor, Gson gson, boolean } } - private void handleNoScanStatusFile(Set scanIdsToWaitFor, boolean shouldWaitAtScanLevel, boolean scassScan, + private void handleNoScanStatusFile(Queue scanIdsToWaitFor, boolean shouldWaitAtScanLevel, boolean scassScan, String scanOutputLocation) { if (scassScan) { String errorMessage = String.format("Unable to find scanOutput.json file at location: {}. Unable to upload BDIO to continue signature scan.", scanOutputLocation); diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/utility/ConcurrentScanWaiter.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/utility/ConcurrentScanWaiter.java new file mode 100644 index 0000000000..b23347325f --- /dev/null +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/utility/ConcurrentScanWaiter.java @@ -0,0 +1,63 @@ +package com.blackduck.integration.detect.lifecycle.run.step.utility; + +import com.blackduck.integration.blackduck.api.generated.view.ProjectVersionView; +import com.blackduck.integration.blackduck.service.model.ProjectVersionWrapper; +import com.blackduck.integration.detect.lifecycle.OperationException; +import com.blackduck.integration.detect.lifecycle.run.data.BlackDuckRunData; +import com.blackduck.integration.detect.lifecycle.run.operation.OperationRunner; +import com.blackduck.integration.rest.HttpUrl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.CompletableFuture; + +public class ConcurrentScanWaiter { + + Logger logger = LoggerFactory.getLogger(ConcurrentScanWaiter.class); + + private final OperationRunner operationRunner; + private final ExecutorService waitingPool; + private final List> activWaits = Collections.synchronizedList(new ArrayList<>()); + + public ConcurrentScanWaiter(ExecutorService waitingPool, OperationRunner operationRunner) { + this.waitingPool = waitingPool; + this.operationRunner = operationRunner; + } + + public void startWaitingForScan(String scanId, BlackDuckRunData blackDuckRunData, ProjectVersionWrapper projectVersion, String threadName) { + if (scanId == null && !blackDuckRunData.shouldWaitAtScanLevel()) return; + + CompletableFuture waitFuture = CompletableFuture.runAsync(() -> { + Thread.currentThread().setName(threadName + " Wait For BOM Thread"); + try { + + if (scanId == null) { + logger.debug("Unexpected null scanID for project version" + projectVersion.getProjectVersionView().getVersionName() + + " skipping waiting for this scan."); + return; + } + + HttpUrl bomUrl = projectVersion.getProjectVersionView().getFirstLink(ProjectVersionView.BOM_STATUS_LINK); + HttpUrl scanUrl = new HttpUrl(bomUrl.toString() + "/" + scanId); + operationRunner.waitForBomCompletion(blackDuckRunData, scanUrl); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, waitingPool); + + activWaits.add(waitFuture); + } + + public void waitForAllScansToComplete() throws OperationException { + try { + CompletableFuture.allOf(activWaits.toArray(new CompletableFuture[0])).get(); + } catch (ExecutionException | InterruptedException e) { + throw new OperationException(e); + } + } +} diff --git a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/utility/StepHelper.java b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/utility/StepHelper.java index 19b856b3ed..6dc7350edc 100644 --- a/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/utility/StepHelper.java +++ b/src/main/java/com/blackduck/integration/detect/lifecycle/run/step/utility/StepHelper.java @@ -1,7 +1,10 @@ package com.blackduck.integration.detect.lifecycle.run.step.utility; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; import java.util.function.Consumer; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +36,21 @@ public void runToolIfIncluded(DetectTool detectTool, String name, OperationWrapp }, () -> {}, (e) -> {}); } + public void runToolIfIncluded(DetectTool detectTool, String name, OperationWrapper.OperationFunction supplier, List> scanFutures, ExecutorService executorService) { + scanFutures.add(CompletableFuture.runAsync(() -> { + Thread.currentThread().setName(name + " Thread"); + try { + runToolIfIncluded(detectTool, name, () -> { + supplier.execute(); + return true; + }, () -> {}, (e) -> {}); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, executorService)); + + } + public Optional runToolIfIncluded(DetectTool detectTool, String name, OperationWrapper.OperationSupplier supplier) throws OperationException { return runToolIfIncluded(detectTool, name, supplier, () -> {}, (e) -> {}); } diff --git a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulator.java b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulator.java index 98eaac8735..f56da8266a 100644 --- a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulator.java +++ b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationAccumulator.java @@ -1,20 +1,19 @@ package com.blackduck.integration.detect.workflow.blackduck.codelocation; -import java.util.ArrayList; -import java.util.EnumMap; -import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import com.blackduck.integration.blackduck.codelocation.CodeLocationBatchOutput; import com.blackduck.integration.blackduck.codelocation.CodeLocationCreationData; import com.blackduck.integration.detect.configuration.enumeration.DetectTool; public class CodeLocationAccumulator { - private final List waitableCodeLocationData = new ArrayList<>(); - private final Set nonWaitableCodeLocations = new HashSet<>(); - private final Map additionalCountsByTool = new EnumMap<>(DetectTool.class); + private final Queue waitableCodeLocationData = new ConcurrentLinkedQueue<>(); + private final Queue nonWaitableCodeLocations = new ConcurrentLinkedQueue<>(); + private final Map additionalCountsByTool = new ConcurrentHashMap<>(); public void addWaitableCodeLocations(DetectTool detectTool, CodeLocationCreationData> creationData) { addWaitableCodeLocations(new WaitableCodeLocationData(detectTool, @@ -40,11 +39,11 @@ public void incrementAdditionalCounts(DetectTool tool, int count) { additionalCountsByTool.put(tool, additionalCountsByTool.getOrDefault(tool, 0) + count); } - public List getWaitableCodeLocations() { + public Queue getWaitableCodeLocations() { return waitableCodeLocationData; } - public Set getNonWaitableCodeLocations() { + public Queue getNonWaitableCodeLocations() { return nonWaitableCodeLocations; } diff --git a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationWaitCalculator.java b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationWaitCalculator.java index c8d24caaf8..46cbedca63 100644 --- a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationWaitCalculator.java +++ b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/codelocation/CodeLocationWaitCalculator.java @@ -2,13 +2,13 @@ import java.util.Date; import java.util.HashSet; -import java.util.List; +import java.util.Queue; import java.util.Set; import com.blackduck.integration.blackduck.service.model.NotificationTaskRange; public class CodeLocationWaitCalculator { - public CodeLocationWaitData calculateWaitData(List codeLocationCreationDatas) { + public CodeLocationWaitData calculateWaitData(Queue codeLocationCreationDatas) { int expectedNotificationCount = 0; NotificationTaskRange notificationRange = null; Set codeLocationNames = new HashSet<>(); diff --git a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/integratedmatching/ScanCountsPayloadCreator.java b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/integratedmatching/ScanCountsPayloadCreator.java index dd1d823dee..2622d07503 100644 --- a/src/main/java/com/blackduck/integration/detect/workflow/blackduck/integratedmatching/ScanCountsPayloadCreator.java +++ b/src/main/java/com/blackduck/integration/detect/workflow/blackduck/integratedmatching/ScanCountsPayloadCreator.java @@ -3,6 +3,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Queue; import org.jetbrains.annotations.NotNull; @@ -13,7 +14,7 @@ public class ScanCountsPayloadCreator { - public ScanCountsPayload create(List createdCodelocations, Map additionalCounts) { + public ScanCountsPayload create(Queue createdCodelocations, Map additionalCounts) { Map countsByTool = collectCountsByTool(createdCodelocations, additionalCounts); return createPayloadFromCountsByTool(countsByTool); } @@ -30,7 +31,7 @@ private ScanCountsPayload createPayloadFromCountsByTool(final Map collectCountsByTool(final List createdCodelocations, Map additionalCounts) { + private Map collectCountsByTool(final Queue createdCodelocations, Map additionalCounts) { Map countsByTool = new HashMap<>(additionalCounts); for (WaitableCodeLocationData waitableCodeLocationData : createdCodelocations) { int oldCount = countsByTool.getOrDefault(waitableCodeLocationData.getDetectTool(), 0); diff --git a/src/main/java/com/blackduck/integration/detect/workflow/diagnostic/DiagnosticLogger.java b/src/main/java/com/blackduck/integration/detect/workflow/diagnostic/DiagnosticLogger.java index 0e4b69176e..8bcaac2c82 100644 --- a/src/main/java/com/blackduck/integration/detect/workflow/diagnostic/DiagnosticLogger.java +++ b/src/main/java/com/blackduck/integration/detect/workflow/diagnostic/DiagnosticLogger.java @@ -53,7 +53,7 @@ private FileAppender addAppender(String file) { LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); PatternLayoutEncoder ple = new PatternLayoutEncoder(); - ple.setPattern("%date{\"yyyy-MM-dd'T'HH:mm:ss,SSSXXX\", UTC} %level [%file:%line] %msg%n"); + ple.setPattern("%date{\"yyyy-MM-dd'T'HH:mm:ss,SSSXXX\", UTC} %level [%thread] [%file:%line] %msg%n"); ple.setContext(lc); ple.start(); diff --git a/src/test/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunnerTest.java b/src/test/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunnerTest.java index e947174251..82e93f9920 100644 --- a/src/test/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunnerTest.java +++ b/src/test/java/com/blackduck/integration/detect/lifecycle/run/step/SignatureScanStepRunnerTest.java @@ -9,6 +9,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.http.HttpHost; import org.apache.http.conn.HttpHostConnectException; @@ -56,7 +58,7 @@ public void setUp() { @Test public void testRunSignatureScannerOnlineRetriesOnHttpHostConnectException() throws Exception { DockerTargetData dockerTargetData = mock(DockerTargetData.class); - Set scanIdsToWaitFor = new HashSet<>(); + Queue scanIdsToWaitFor = new ConcurrentLinkedQueue<>(); List scanPaths = Collections.singletonList(mock(SignatureScanPath.class)); ScanBatch scanBatch = mock(ScanBatch.class); @@ -82,7 +84,7 @@ public void testRunSignatureScannerOnlineRetriesOnHttpHostConnectException() thr @Test public void testRunSignatureScannerOnlineExecutesOnceWithoutException() throws Exception { DockerTargetData dockerTargetData = mock(DockerTargetData.class); - Set scanIdsToWaitFor = new HashSet<>(); + Queue scanIdsToWaitFor = new ConcurrentLinkedQueue<>(); List scanPaths = Collections.singletonList(mock(SignatureScanPath.class)); ScanBatch scanBatch = mock(ScanBatch.class); diff --git a/src/test/java/com/blackduck/integration/detect/workflow/blackduck/integratedmatching/ScanCountsPayloadCreatorTest.java b/src/test/java/com/blackduck/integration/detect/workflow/blackduck/integratedmatching/ScanCountsPayloadCreatorTest.java index cb5589131b..a055315f4d 100644 --- a/src/test/java/com/blackduck/integration/detect/workflow/blackduck/integratedmatching/ScanCountsPayloadCreatorTest.java +++ b/src/test/java/com/blackduck/integration/detect/workflow/blackduck/integratedmatching/ScanCountsPayloadCreatorTest.java @@ -2,12 +2,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import java.util.Arrays; -import java.util.HashMap; +import java.util.Queue; import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Set; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -35,9 +36,8 @@ void testMultSigScansPlusBinary() { successfulBinaryCodeLocationNames.add("binaryScan3"); Mockito.when(binaryWaitableCodeLocationData.getSuccessfulCodeLocationNames()).thenReturn(successfulBinaryCodeLocationNames); - List waitableCodeLocationDataList = Arrays.asList( - signatureScanWaitableCodeLocationData, - binaryWaitableCodeLocationData); + Queue waitableCodeLocationDataList = new ConcurrentLinkedQueue<>(); + Collections.addAll(waitableCodeLocationDataList, signatureScanWaitableCodeLocationData, binaryWaitableCodeLocationData); ScanCountsPayloadCreator creator = new ScanCountsPayloadCreator(); @@ -82,11 +82,8 @@ void testAllPkgMgrTypesPlusIgnored() { successfulIacCodeLocationNames.add("iacScan2"); Mockito.when(iacWaitableCodeLocationData.getSuccessfulCodeLocationNames()).thenReturn(successfulIacCodeLocationNames); - List waitableCodeLocationDataList = Arrays.asList( - bazelWaitableCodeLocationData, - dockerWaitableCodeLocationData, - detectorWaitableCodeLocationData, - iacWaitableCodeLocationData); + Queue waitableCodeLocationDataList = new ConcurrentLinkedQueue<>(); + Collections.addAll(waitableCodeLocationDataList,bazelWaitableCodeLocationData, detectorWaitableCodeLocationData, iacWaitableCodeLocationData, dockerWaitableCodeLocationData); ScanCountsPayloadCreator creator = new ScanCountsPayloadCreator(); ScanCountsPayload payload = creator.create(waitableCodeLocationDataList, new HashMap<>());