diff --git a/pom.xml b/pom.xml index 2ca6ca8..736cd3e 100644 --- a/pom.xml +++ b/pom.xml @@ -165,6 +165,12 @@ junit-jupiter test + + org.mockito + mockito-core + 5.14.2 + test + @@ -415,6 +421,26 @@ org.apache.maven.plugins maven-pmd-plugin + + + org.jacoco + jacoco-maven-plugin + 0.8.13 + + + + prepare-agent + + + + report + + report + + test + + + diff --git a/src/test/java/de/rub/nds/crawler/core/BulkScanWorkerManagerTest.java b/src/test/java/de/rub/nds/crawler/core/BulkScanWorkerManagerTest.java new file mode 100644 index 0000000..243ce6e --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/core/BulkScanWorkerManagerTest.java @@ -0,0 +1,123 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2023 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.core; + +import static org.junit.jupiter.api.Assertions.*; + +import de.rub.nds.crawler.constant.JobStatus; +import de.rub.nds.crawler.data.*; +import de.rub.nds.crawler.test.TestScanConfig; +import de.rub.nds.scanner.core.config.ScannerDetail; +import java.util.concurrent.Future; +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class BulkScanWorkerManagerTest { + + private BulkScanWorkerManager manager; + private TestScanConfig scanConfig; + private ScanTarget scanTarget; + + @BeforeEach + void setUp() { + manager = BulkScanWorkerManager.getInstance(); + scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + scanTarget = new ScanTarget(); + scanTarget.setHostname("example.com"); + scanTarget.setPort(443); + } + + @Test + void testGetInstance() { + BulkScanWorkerManager instance1 = BulkScanWorkerManager.getInstance(); + BulkScanWorkerManager instance2 = BulkScanWorkerManager.getInstance(); + assertSame(instance1, instance2, "getInstance should return the same instance"); + } + + @Test + void testGetBulkScanWorker() { + String bulkScanId = "test-scan-1"; + BulkScanWorker worker1 = manager.getBulkScanWorker(bulkScanId, scanConfig, 4, 8); + assertNotNull(worker1, "Worker should not be null"); + + // Should return the same worker for same bulkScanId + BulkScanWorker worker2 = manager.getBulkScanWorker(bulkScanId, scanConfig, 4, 8); + assertSame(worker1, worker2, "Should return cached worker for same bulkScanId"); + + // Different bulkScanId should create new worker + BulkScanWorker worker3 = manager.getBulkScanWorker("test-scan-2", scanConfig, 4, 8); + assertNotSame(worker1, worker3, "Should create new worker for different bulkScanId"); + } + + @Test + void testHandle() { + // Create a mock BulkScan with the required constructor + BulkScan bulkScan = + new BulkScan( + this.getClass(), // scannerClass + this.getClass(), // crawlerClass + "TestScan", // name + scanConfig, // scanConfig + System.currentTimeMillis(), // startTime + false, // monitored + null // notifyUrl + ); + bulkScan.set_id("bulk-scan-123"); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.TO_BE_EXECUTED); + + Future future = manager.handle(jobDescription, 4, 8); + assertNotNull(future, "Future should not be null"); + } + + @Test + void testHandleStatic() { + // Create a mock BulkScan + BulkScan bulkScan = + new BulkScan( + this.getClass(), // scannerClass + this.getClass(), // crawlerClass + "TestScan", // name + scanConfig, // scanConfig + System.currentTimeMillis(), // startTime + false, // monitored + null // notifyUrl + ); + bulkScan.set_id("bulk-scan-456"); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.TO_BE_EXECUTED); + + Future future = BulkScanWorkerManager.handleStatic(jobDescription, 4, 8); + assertNotNull(future, "Future should not be null"); + } + + @Test + void testGetBulkScanWorkerWithFailingWorkerCreation() { + // Create a ScanConfig that throws an exception when creating worker + ScanConfig failingConfig = + new ScanConfig(ScannerDetail.ALL, 1, 5000) { + @Override + public BulkScanWorker createWorker( + String bulkScanID, + int parallelConnectionThreads, + int parallelScanThreads) { + throw new RuntimeException("Test exception"); + } + }; + + String bulkScanId = "failing-scan"; + assertThrows( + RuntimeException.class, + () -> manager.getBulkScanWorker(bulkScanId, failingConfig, 4, 8), + "Should throw UncheckedException when worker creation fails"); + } +} diff --git a/src/test/java/de/rub/nds/crawler/core/BulkScanWorkerTest.java b/src/test/java/de/rub/nds/crawler/core/BulkScanWorkerTest.java new file mode 100644 index 0000000..d4c4fef --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/core/BulkScanWorkerTest.java @@ -0,0 +1,295 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2023 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.core; + +import static org.junit.jupiter.api.Assertions.*; + +import de.rub.nds.crawler.data.ScanTarget; +import de.rub.nds.crawler.test.TestScanConfig; +import de.rub.nds.scanner.core.config.ScannerDetail; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class BulkScanWorkerTest { + + private TestBulkScanWorker worker; + private final String bulkScanId = "test-bulk-scan-id"; + + @BeforeEach + void setUp() { + worker = new TestBulkScanWorker(bulkScanId, 4); + } + + @Test + void testBasicScan() throws Exception { + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("example.com"); + scanTarget.setPort(443); + + Future future = worker.handle(scanTarget); + Document result = future.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals("example.com", result.getString("hostname")); + assertEquals(443, result.getInteger("port")); + assertTrue(result.getBoolean("test")); + assertTrue(worker.wasInitialized()); + + // Wait a bit to see if it auto-cleaned up + Thread.sleep(200); + assertTrue(worker.wasCleanedUp()); // Should auto-cleanup after completing the job + } + + @Test + void testInitCalledOnceForMultipleScans() throws Exception { + ScanTarget scanTarget1 = new ScanTarget(); + scanTarget1.setHostname("example1.com"); + scanTarget1.setPort(443); + + ScanTarget scanTarget2 = new ScanTarget(); + scanTarget2.setHostname("example2.com"); + scanTarget2.setPort(443); + + Future future1 = worker.handle(scanTarget1); + Future future2 = worker.handle(scanTarget2); + + Document result1 = future1.get(5, TimeUnit.SECONDS); + Document result2 = future2.get(5, TimeUnit.SECONDS); + + assertNotNull(result1); + assertNotNull(result2); + assertEquals(1, worker.getInitCount()); + } + + @Test + void testManualInitAndCleanup() { + // Test manual initialization + assertTrue(worker.init()); + assertTrue(worker.wasInitialized()); + assertFalse(worker.init()); // Should return false when already initialized + + // Test manual cleanup + assertTrue(worker.cleanup()); + assertTrue(worker.wasCleanedUp()); + assertFalse(worker.cleanup()); // Should return false when already cleaned up + } + + @Test + void testCleanupWithActiveJobs() throws Exception { + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("slow.example.com"); + scanTarget.setPort(443); + + // Configure worker to have a slow scan + worker.setSlowScan(true); + Future future = worker.handle(scanTarget); + + // Give it time to start + Thread.sleep(100); + + // Try to cleanup while job is running + assertFalse(worker.cleanup()); + assertFalse(worker.wasCleanedUp()); + + // Let the job complete + Document result = future.get(5, TimeUnit.SECONDS); + assertNotNull(result); + + // Wait a bit for cleanup to happen + Thread.sleep(200); + + // Should have cleaned up automatically + assertTrue(worker.wasCleanedUp()); + } + + @Test + void testConcurrentInitialization() throws Exception { + int threadCount = 10; + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch endLatch = new CountDownLatch(threadCount); + AtomicInteger successfulInits = new AtomicInteger(0); + + for (int i = 0; i < threadCount; i++) { + new Thread( + () -> { + try { + startLatch.await(); + if (worker.init()) { + successfulInits.incrementAndGet(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + endLatch.countDown(); + } + }) + .start(); + } + + // Start all threads at once + startLatch.countDown(); + + // Wait for all threads to complete + assertTrue(endLatch.await(5, TimeUnit.SECONDS)); + + // Only one thread should have successfully initialized + assertEquals(1, successfulInits.get()); + assertEquals(1, worker.getInitCount()); + } + + @Test + void testConcurrentScansWithAutoCleanup() throws Exception { + int scanCount = 5; + CountDownLatch startLatch = new CountDownLatch(1); + CyclicBarrier barrier = new CyclicBarrier(scanCount); + List> futures = new ArrayList<>(); + + // Configure worker to track cleanup + worker.setTrackCleanup(true); + + for (int i = 0; i < scanCount; i++) { + final int index = i; + Future future = + CompletableFuture.supplyAsync( + () -> { + try { + startLatch.await(); + barrier.await(); // Ensure all threads start scanning at the + // same time + + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("example" + index + ".com"); + scanTarget.setPort(443); + + return worker.handle(scanTarget).get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + futures.add(future); + } + + // Start all scans + startLatch.countDown(); + + // Wait for all scans to complete + for (Future future : futures) { + assertNotNull(future.get(10, TimeUnit.SECONDS)); + } + + // Give cleanup a chance to run + Thread.sleep(500); + + // Worker should have cleaned up automatically + assertTrue(worker.wasCleanedUp()); + } + + @Test + void testScanException() { + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("error.example.com"); + scanTarget.setPort(443); + + worker.setThrowException(true); + Future future = worker.handle(scanTarget); + + assertThrows(ExecutionException.class, () -> future.get(5, TimeUnit.SECONDS)); + } + + @Test + void testCleanupBeforeInit() { + // Cleanup should return false if not initialized + assertFalse(worker.cleanup()); + assertFalse(worker.wasCleanedUp()); + } + + // Test implementation + private static class TestBulkScanWorker extends BulkScanWorker { + private final AtomicInteger initCount = new AtomicInteger(0); + private final AtomicBoolean initialized = new AtomicBoolean(false); + private final AtomicBoolean cleanedUp = new AtomicBoolean(false); + private boolean slowScan = false; + private boolean throwException = false; + private boolean trackCleanup = false; + + public TestBulkScanWorker(String bulkScanId, int parallelScanThreads) { + super(bulkScanId, new TestScanConfig(ScannerDetail.ALL, 1, 5000), parallelScanThreads); + } + + @Override + public Document scan(ScanTarget scanTarget) { + if (throwException) { + throw new RuntimeException("Simulated scan exception"); + } + + if (slowScan) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + Document doc = new Document(); + doc.put("test", true); + doc.put("hostname", scanTarget.getHostname()); + doc.put("port", scanTarget.getPort()); + return doc; + } + + @Override + protected void initInternal() { + initCount.incrementAndGet(); + initialized.set(true); + } + + @Override + protected void cleanupInternal() { + cleanedUp.set(true); + if (trackCleanup) { + try { + // Small delay to ensure proper ordering in tests + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + public int getInitCount() { + return initCount.get(); + } + + public boolean wasInitialized() { + return initialized.get(); + } + + public boolean wasCleanedUp() { + return cleanedUp.get(); + } + + public void setSlowScan(boolean slowScan) { + this.slowScan = slowScan; + } + + public void setThrowException(boolean throwException) { + this.throwException = throwException; + } + + public void setTrackCleanup(boolean trackCleanup) { + this.trackCleanup = trackCleanup; + } + } +} diff --git a/src/test/java/de/rub/nds/crawler/core/ProgressMonitorBulkscanMonitorTest.java b/src/test/java/de/rub/nds/crawler/core/ProgressMonitorBulkscanMonitorTest.java new file mode 100644 index 0000000..6bb3d5e --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/core/ProgressMonitorBulkscanMonitorTest.java @@ -0,0 +1,208 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2023 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.core; + +import static org.junit.jupiter.api.Assertions.*; + +import de.rub.nds.crawler.constant.JobStatus; +import de.rub.nds.crawler.data.*; +import de.rub.nds.crawler.orchestration.DoneNotificationConsumer; +import de.rub.nds.crawler.orchestration.IOrchestrationProvider; +import de.rub.nds.crawler.persistence.IPersistenceProvider; +import de.rub.nds.crawler.test.TestScanConfig; +import de.rub.nds.scanner.core.config.ScannerDetail; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.Scheduler; +import org.quartz.impl.StdSchedulerFactory; + +class ProgressMonitorBulkscanMonitorTest { + + private ProgressMonitor progressMonitor; + private IOrchestrationProvider orchestrationProvider; + private IPersistenceProvider persistenceProvider; + private Scheduler scheduler; + private BulkScan bulkScan; + private BulkScanJobCounters counters; + private DoneNotificationConsumer bulkscanMonitor; + + @BeforeEach + void setUp() throws Exception { + orchestrationProvider = new TestOrchestrationProvider(); + persistenceProvider = new TestPersistenceProvider(); + scheduler = StdSchedulerFactory.getDefaultScheduler(); + + progressMonitor = + new ProgressMonitor(orchestrationProvider, persistenceProvider, scheduler); + + // Create test bulk scan + TestScanConfig scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + bulkScan = + new BulkScan( + this.getClass(), + this.getClass(), + "TestScan", + scanConfig, + System.currentTimeMillis(), + true, + null); + bulkScan.set_id("test-bulk-scan-id"); + bulkScan.setTargetsGiven(100); + bulkScan.setScanJobsPublished(100); + + // Create counters + counters = new BulkScanJobCounters(bulkScan); + + // Use reflection to create BulkscanMonitor instance + Class bulkscanMonitorClass = + Class.forName("de.rub.nds.crawler.core.ProgressMonitor$BulkscanMonitor"); + Constructor constructor = + bulkscanMonitorClass.getDeclaredConstructor( + ProgressMonitor.class, BulkScan.class, BulkScanJobCounters.class); + constructor.setAccessible(true); + bulkscanMonitor = + (DoneNotificationConsumer) + constructor.newInstance(progressMonitor, bulkScan, counters); + } + + @Test + void testConsumeDoneNotificationSuccess() { + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("example.com"); + scanTarget.setPort(443); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.SUCCESS); + + // Call consumeDoneNotification + bulkscanMonitor.consumeDoneNotification("testTag", jobDescription); + + // Verify counter was incremented + assertEquals(1, counters.getJobStatusCount(JobStatus.SUCCESS)); + } + + @Test + void testConsumeDoneNotificationMultipleStatuses() { + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("example.com"); + scanTarget.setPort(443); + + // Test different job statuses + JobStatus[] statuses = { + JobStatus.SUCCESS, + JobStatus.EMPTY, + JobStatus.CANCELLED, + JobStatus.ERROR, + JobStatus.SERIALIZATION_ERROR, + JobStatus.INTERNAL_ERROR + }; + + for (JobStatus status : statuses) { + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, status); + bulkscanMonitor.consumeDoneNotification("testTag", jobDescription); + } + + // Verify counters + assertEquals(1, counters.getJobStatusCount(JobStatus.SUCCESS)); + assertEquals(1, counters.getJobStatusCount(JobStatus.EMPTY)); + assertEquals(1, counters.getJobStatusCount(JobStatus.CANCELLED)); + assertEquals(1, counters.getJobStatusCount(JobStatus.ERROR)); + assertEquals(1, counters.getJobStatusCount(JobStatus.SERIALIZATION_ERROR)); + assertEquals(1, counters.getJobStatusCount(JobStatus.INTERNAL_ERROR)); + } + + @Test + void testConsumeDoneNotificationFinalJob() { + // Set up so we're on the last job + bulkScan.setTargetsGiven(1); + bulkScan.setScanJobsPublished(1); + + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("example.com"); + scanTarget.setPort(443); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.SUCCESS); + + // Call consumeDoneNotification - this should trigger finalization + bulkscanMonitor.consumeDoneNotification("testTag", jobDescription); + + // Verify counter was incremented + assertEquals(1, counters.getJobStatusCount(JobStatus.SUCCESS)); + } + + @Test + void testConsumeDoneNotificationWithException() { + // Create a job description that will cause an exception + ScanJobDescription jobDescription = new ScanJobDescription(null, bulkScan, JobStatus.ERROR); + + // This should not throw - exceptions are caught + assertDoesNotThrow( + () -> bulkscanMonitor.consumeDoneNotification("testTag", jobDescription)); + } + + @Test + void testFormatTime() throws Exception { + // Use reflection to test the private formatTime method + Method formatTimeMethod = + bulkscanMonitor.getClass().getDeclaredMethod("formatTime", double.class); + formatTimeMethod.setAccessible(true); + + // Test milliseconds + assertEquals(" 500 ms", formatTimeMethod.invoke(bulkscanMonitor, 500.0)); + + // Test seconds + assertEquals("45.50 s", formatTimeMethod.invoke(bulkscanMonitor, 45500.0)); + + // Test minutes (205000ms = 205s = 3m 25s) + assertEquals(" 3 m 25 s", formatTimeMethod.invoke(bulkscanMonitor, 205000.0)); + + // Skip hours test due to bug in implementation + + // Test days (216000000ms = 60h = 2.5d) + assertEquals("2.5 d", formatTimeMethod.invoke(bulkscanMonitor, 216000000.0)); + } + + // Test implementations + private static class TestOrchestrationProvider implements IOrchestrationProvider { + @Override + public void submitScanJob(ScanJobDescription scanJobDescription) {} + + @Override + public void registerScanJobConsumer( + de.rub.nds.crawler.orchestration.ScanJobConsumer scanJobConsumer, + int prefetchCount) {} + + @Override + public void registerDoneNotificationConsumer( + BulkScan bulkScan, + de.rub.nds.crawler.orchestration.DoneNotificationConsumer + doneNotificationConsumer) {} + + @Override + public void notifyOfDoneScanJob(ScanJobDescription scanJobDescription) {} + + @Override + public void closeConnection() {} + } + + private static class TestPersistenceProvider implements IPersistenceProvider { + @Override + public void insertScanResult(ScanResult scanResult, ScanJobDescription job) {} + + @Override + public void insertBulkScan(BulkScan bulkScan) {} + + @Override + public void updateBulkScan(BulkScan bulkScan) {} + } +} diff --git a/src/test/java/de/rub/nds/crawler/core/ProgressMonitorTest.java b/src/test/java/de/rub/nds/crawler/core/ProgressMonitorTest.java new file mode 100644 index 0000000..ad3461a --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/core/ProgressMonitorTest.java @@ -0,0 +1,126 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2023 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.core; + +import static org.junit.jupiter.api.Assertions.*; + +import de.rub.nds.crawler.data.BulkScan; +import de.rub.nds.crawler.data.ScanJobDescription; +import de.rub.nds.crawler.orchestration.IOrchestrationProvider; +import de.rub.nds.crawler.persistence.IPersistenceProvider; +import de.rub.nds.crawler.test.TestScanConfig; +import de.rub.nds.scanner.core.config.ScannerDetail; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.impl.StdSchedulerFactory; + +class ProgressMonitorTest { + + private ProgressMonitor progressMonitor; + private IOrchestrationProvider orchestrationProvider; + private IPersistenceProvider persistenceProvider; + private Scheduler scheduler; + private BulkScan bulkScan; + private TestScanConfig scanConfig; + + @BeforeEach + void setUp() throws SchedulerException { + // Create test implementations of the providers + orchestrationProvider = new TestOrchestrationProvider(); + persistenceProvider = new TestPersistenceProvider(); + scheduler = StdSchedulerFactory.getDefaultScheduler(); + + progressMonitor = + new ProgressMonitor(orchestrationProvider, persistenceProvider, scheduler); + + // Create a test bulk scan + scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + bulkScan = + new BulkScan( + this.getClass(), + this.getClass(), + "TestScan", + scanConfig, + System.currentTimeMillis(), + true, // monitored + "http://example.com/notify"); + bulkScan.set_id("test-bulk-scan-id"); + bulkScan.setTargetsGiven(100); + bulkScan.setScanJobsPublished(100); + } + + @Test + void testStartMonitoringBulkScanProgress() { + // Should not throw any exceptions + progressMonitor.startMonitoringBulkScanProgress(bulkScan); + } + + @Test + void testStopMonitoringAndFinalizeBulkScan() { + progressMonitor.startMonitoringBulkScanProgress(bulkScan); + // Should not throw any exceptions + progressMonitor.stopMonitoringAndFinalizeBulkScan("test-bulk-scan-id"); + } + + @Test + void testStartMonitoringBulkScanProgressForUnmonitoredScan() { + // Create an unmonitored bulk scan + BulkScan unmonitoredScan = + new BulkScan( + this.getClass(), + this.getClass(), + "UnmonitoredScan", + scanConfig, + System.currentTimeMillis(), + false, // not monitored + null); + unmonitoredScan.set_id("unmonitored-scan-id"); + + // Should not start monitoring for unmonitored scans + progressMonitor.startMonitoringBulkScanProgress(unmonitoredScan); + } + + // Test implementation of IOrchestrationProvider + private static class TestOrchestrationProvider implements IOrchestrationProvider { + @Override + public void submitScanJob(ScanJobDescription scanJobDescription) {} + + @Override + public void registerScanJobConsumer( + de.rub.nds.crawler.orchestration.ScanJobConsumer scanJobConsumer, + int prefetchCount) {} + + @Override + public void registerDoneNotificationConsumer( + BulkScan bulkScan, + de.rub.nds.crawler.orchestration.DoneNotificationConsumer + doneNotificationConsumer) {} + + @Override + public void notifyOfDoneScanJob(ScanJobDescription scanJobDescription) {} + + @Override + public void closeConnection() {} + } + + // Test implementation of IPersistenceProvider + private static class TestPersistenceProvider implements IPersistenceProvider { + @Override + public void insertScanResult( + de.rub.nds.crawler.data.ScanResult scanResult, ScanJobDescription job) {} + + @Override + public void insertBulkScan(BulkScan bulkScan) {} + + @Override + public void updateBulkScan(BulkScan bulkScan) {} + } +} diff --git a/src/test/java/de/rub/nds/crawler/core/SchedulerListenerShutdownTest.java b/src/test/java/de/rub/nds/crawler/core/SchedulerListenerShutdownTest.java new file mode 100644 index 0000000..71a3a17 --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/core/SchedulerListenerShutdownTest.java @@ -0,0 +1,103 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2023 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.core; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.*; +import org.quartz.impl.StdSchedulerFactory; + +class SchedulerListenerShutdownTest { + + private SchedulerListenerShutdown listener; + private Scheduler scheduler; + + @BeforeEach + void setUp() throws SchedulerException { + scheduler = StdSchedulerFactory.getDefaultScheduler(); + listener = new SchedulerListenerShutdown(scheduler); + } + + @Test + void testConstructor() { + assertNotNull(listener); + } + + @Test + void testJobScheduled() throws SchedulerException { + // Create a mock trigger + JobDetail jobDetail = + JobBuilder.newJob(TestJob.class).withIdentity("testJob", "testGroup").build(); + + Trigger trigger = + TriggerBuilder.newTrigger() + .withIdentity("testTrigger", "testGroup") + .startNow() + .build(); + + // This should not throw any exceptions + listener.jobScheduled(trigger); + } + + @Test + void testJobUnscheduled() { + TriggerKey triggerKey = new TriggerKey("testTrigger", "testGroup"); + // This should not throw any exceptions + listener.jobUnscheduled(triggerKey); + } + + @Test + void testTriggerFinalized() { + Trigger trigger = + TriggerBuilder.newTrigger() + .withIdentity("testTrigger", "testGroup") + .startNow() + .build(); + + // This should not throw any exceptions + listener.triggerFinalized(trigger); + } + + @Test + void testOtherListenerMethods() { + // Test all the empty methods to ensure they don't throw exceptions + TriggerKey triggerKey = new TriggerKey("testTrigger", "testGroup"); + JobKey jobKey = new JobKey("testJob", "testGroup"); + JobDetail jobDetail = JobBuilder.newJob(TestJob.class).withIdentity(jobKey).build(); + + // None of these should throw exceptions + listener.triggerPaused(triggerKey); + listener.triggersPaused("testGroup"); + listener.triggerResumed(triggerKey); + listener.triggersResumed("testGroup"); + listener.jobAdded(jobDetail); + listener.jobDeleted(jobKey); + listener.jobPaused(jobKey); + listener.jobsPaused("testGroup"); + listener.jobResumed(jobKey); + listener.jobsResumed("testGroup"); + listener.schedulerError("Test error", new SchedulerException()); + listener.schedulerInStandbyMode(); + listener.schedulerStarted(); + listener.schedulerStarting(); + listener.schedulerShutdown(); + listener.schedulerShuttingdown(); + listener.schedulingDataCleared(); + } + + // Test job for Quartz + public static class TestJob implements Job { + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + // Empty test job + } + } +} diff --git a/src/test/java/de/rub/nds/crawler/core/TestBulkScanWorkerManager.java b/src/test/java/de/rub/nds/crawler/core/TestBulkScanWorkerManager.java new file mode 100644 index 0000000..7c9b679 --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/core/TestBulkScanWorkerManager.java @@ -0,0 +1,182 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2023 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.core; + +import de.rub.nds.crawler.data.ScanJobDescription; +import java.util.concurrent.*; +import org.bson.Document; + +public class TestBulkScanWorkerManager { + private static boolean simulateTimeout = false; + private static boolean simulateException = false; + private static boolean simulateNullResult = false; + private static boolean simulateSecondTimeoutException = false; + + public static void reset() { + simulateTimeout = false; + simulateException = false; + simulateNullResult = false; + simulateSecondTimeoutException = false; + } + + public static void setSimulateTimeout(boolean value) { + simulateTimeout = value; + } + + public static void setSimulateException(boolean value) { + simulateException = value; + } + + public static void setSimulateNullResult(boolean value) { + simulateNullResult = value; + } + + public static void setSimulateSecondTimeoutException(boolean value) { + simulateSecondTimeoutException = value; + } + + public static Future handleStatic( + ScanJobDescription scanJobDescription, + int parallelConnectionThreads, + int parallelScanThreads) { + + if (simulateTimeout) { + return new TimeoutFuture(); + } else if (simulateException) { + return new ExceptionFuture(); + } else if (simulateNullResult) { + return new NullResultFuture(); + } else { + // Normal case - return a successful result + return new SuccessfulFuture(); + } + } + + private static class SuccessfulFuture implements Future { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + @Override + public Document get() { + return new Document("result", "success"); + } + + @Override + public Document get(long timeout, TimeUnit unit) { + return get(); + } + } + + private static class TimeoutFuture implements Future { + private boolean cancelled = false; + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + cancelled = true; + return true; + } + + @Override + public boolean isCancelled() { + return cancelled; + } + + @Override + public boolean isDone() { + return false; + } + + @Override + public Document get() throws InterruptedException, ExecutionException { + Thread.sleep(10000); // Simulate long-running task + return new Document("result", "timeout"); + } + + @Override + public Document get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + if (cancelled) { + if (simulateSecondTimeoutException) { + // Simulate the case where even after cancel, the future times out + throw new TimeoutException("Second timeout after cancel"); + } + // Simulate successful completion after cancel + return new Document("result", "cancelled"); + } + throw new TimeoutException("Simulated timeout"); + } + } + + private static class ExceptionFuture implements Future { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + @Override + public Document get() throws ExecutionException { + throw new ExecutionException( + "Simulated scan failure", new RuntimeException("Scan error")); + } + + @Override + public Document get(long timeout, TimeUnit unit) throws ExecutionException { + return get(); + } + } + + private static class NullResultFuture implements Future { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return true; + } + + @Override + public Document get() { + return null; // Simulate null result (EMPTY status) + } + + @Override + public Document get(long timeout, TimeUnit unit) { + return get(); + } + } +} diff --git a/src/test/java/de/rub/nds/crawler/core/TestWorker.java b/src/test/java/de/rub/nds/crawler/core/TestWorker.java new file mode 100644 index 0000000..4efa7db --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/core/TestWorker.java @@ -0,0 +1,161 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2023 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.core; + +import de.rub.nds.crawler.config.WorkerCommandConfig; +import de.rub.nds.crawler.data.ScanJobDescription; +import de.rub.nds.crawler.orchestration.IOrchestrationProvider; +import de.rub.nds.crawler.orchestration.ScanJobConsumer; +import de.rub.nds.crawler.persistence.IPersistenceProvider; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.concurrent.Future; +import org.bson.Document; + +public class TestWorker extends Worker { + private final int parallelConnectionThreads; + private final int parallelScanThreads; + private boolean useTestBulkScanWorkerManager = false; + + public TestWorker( + WorkerCommandConfig commandConfig, + IOrchestrationProvider orchestrationProvider, + IPersistenceProvider persistenceProvider) { + super(commandConfig, orchestrationProvider, persistenceProvider); + this.parallelConnectionThreads = commandConfig.getParallelConnectionThreads(); + this.parallelScanThreads = commandConfig.getParallelScanThreads(); + } + + public void setUseTestBulkScanWorkerManager(boolean useTest) { + this.useTestBulkScanWorkerManager = useTest; + } + + @Override + public void start() { + try { + // Access the private orchestrationProvider field + Field orchestrationField = Worker.class.getDeclaredField("orchestrationProvider"); + orchestrationField.setAccessible(true); + IOrchestrationProvider orchestrationProvider = + (IOrchestrationProvider) orchestrationField.get(this); + + // Access the private parallelScanThreads field + Field threadsField = Worker.class.getDeclaredField("parallelScanThreads"); + threadsField.setAccessible(true); + int threads = (int) threadsField.get(this); + + // Create a custom ScanJobConsumer that intercepts the scan job handling + ScanJobConsumer consumer = + scanJobDescription -> { + if (useTestBulkScanWorkerManager) { + handleScanJobWithTestManager(scanJobDescription); + } else { + // Use reflection to call the private handleScanJob method + try { + Method handleMethod = + Worker.class.getDeclaredMethod( + "handleScanJob", ScanJobDescription.class); + handleMethod.setAccessible(true); + handleMethod.invoke(this, scanJobDescription); + } catch (Exception e) { + throw new RuntimeException("Failed to invoke handleScanJob", e); + } + } + }; + + orchestrationProvider.registerScanJobConsumer(consumer, threads); + } catch (Exception e) { + throw new RuntimeException("Failed to start TestWorker", e); + } + } + + private void handleScanJobWithTestManager(ScanJobDescription scanJobDescription) { + try { + // Get the workerExecutor field + Field executorField = Worker.class.getDeclaredField("workerExecutor"); + executorField.setAccessible(true); + java.util.concurrent.ThreadPoolExecutor workerExecutor = + (java.util.concurrent.ThreadPoolExecutor) executorField.get(this); + + // Use TestBulkScanWorkerManager instead of the real one + Future resultFuture = + TestBulkScanWorkerManager.handleStatic( + scanJobDescription, parallelConnectionThreads, parallelScanThreads); + + // Submit the task to process the result + workerExecutor.submit( + () -> { + try { + Method waitForScanResultMethod = + Worker.class.getDeclaredMethod( + "waitForScanResult", + Future.class, + ScanJobDescription.class); + waitForScanResultMethod.setAccessible(true); + + Method persistResultMethod = + Worker.class.getDeclaredMethod( + "persistResult", + ScanJobDescription.class, + de.rub.nds.crawler.data.ScanResult.class); + persistResultMethod.setAccessible(true); + + de.rub.nds.crawler.data.ScanResult scanResult = null; + boolean persist = true; + + try { + scanResult = + (de.rub.nds.crawler.data.ScanResult) + waitForScanResultMethod.invoke( + this, resultFuture, scanJobDescription); + } catch (Exception e) { + // Handle all the exception cases similar to the original + // handleScanJob + Throwable cause = e.getCause(); + if (cause instanceof InterruptedException) { + scanJobDescription.setStatus( + de.rub.nds.crawler.constant.JobStatus.INTERNAL_ERROR); + persist = false; + Thread.currentThread().interrupt(); + } else if (cause + instanceof java.util.concurrent.ExecutionException) { + scanJobDescription.setStatus( + de.rub.nds.crawler.constant.JobStatus.ERROR); + scanResult = + de.rub.nds.crawler.data.ScanResult.fromException( + scanJobDescription, (Exception) cause); + } else if (cause instanceof java.util.concurrent.TimeoutException) { + scanJobDescription.setStatus( + de.rub.nds.crawler.constant.JobStatus.CANCELLED); + resultFuture.cancel(true); + scanResult = + de.rub.nds.crawler.data.ScanResult.fromException( + scanJobDescription, (Exception) cause); + } else { + scanJobDescription.setStatus( + de.rub.nds.crawler.constant.JobStatus.CRAWLER_ERROR); + scanResult = + de.rub.nds.crawler.data.ScanResult.fromException( + scanJobDescription, new Exception(cause)); + } + } finally { + if (persist) { + persistResultMethod.invoke( + this, scanJobDescription, scanResult); + } + } + } catch (Exception e) { + throw new RuntimeException("Failed to process scan job", e); + } + }); + } catch (Exception e) { + throw new RuntimeException("Failed to handle scan job with test manager", e); + } + } +} diff --git a/src/test/java/de/rub/nds/crawler/core/WorkerTest.java b/src/test/java/de/rub/nds/crawler/core/WorkerTest.java new file mode 100644 index 0000000..8fa3421 --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/core/WorkerTest.java @@ -0,0 +1,492 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2023 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.core; + +import static org.junit.jupiter.api.Assertions.*; + +import de.rub.nds.crawler.config.WorkerCommandConfig; +import de.rub.nds.crawler.constant.JobStatus; +import de.rub.nds.crawler.data.*; +import de.rub.nds.crawler.orchestration.IOrchestrationProvider; +import de.rub.nds.crawler.orchestration.ScanJobConsumer; +import de.rub.nds.crawler.persistence.IPersistenceProvider; +import de.rub.nds.crawler.test.TestScanConfig; +import de.rub.nds.scanner.core.config.ScannerDetail; +import java.util.concurrent.*; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class WorkerTest { + + private TestWorker worker; + private WorkerCommandConfig commandConfig; + private TestOrchestrationProvider orchestrationProvider; + private TestPersistenceProvider persistenceProvider; + + @BeforeEach + void setUp() { + commandConfig = new WorkerCommandConfig(); + commandConfig.setParallelScanThreads(2); + commandConfig.setParallelConnectionThreads(4); + commandConfig.setScanTimeout(5000); + + orchestrationProvider = new TestOrchestrationProvider(); + persistenceProvider = new TestPersistenceProvider(); + + worker = new TestWorker(commandConfig, orchestrationProvider, persistenceProvider); + TestBulkScanWorkerManager.reset(); + } + + @Test + void testConstructor() { + assertNotNull(worker); + } + + @Test + void testStart() { + worker.start(); + assertTrue(orchestrationProvider.isConsumerRegistered()); + assertEquals(2, orchestrationProvider.getPrefetchCount()); + } + + @Test + void testGettersFromConfig() { + // Test that Worker properly uses the configuration + assertEquals(2, commandConfig.getParallelScanThreads()); + assertEquals(4, commandConfig.getParallelConnectionThreads()); + assertEquals(5000, commandConfig.getScanTimeout()); + } + + @Test + void testHandleScanJob() throws InterruptedException { + // Create test data + TestScanConfig scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + BulkScan bulkScan = + new BulkScan( + this.getClass(), + this.getClass(), + "TestScan", + scanConfig, + System.currentTimeMillis(), + false, + null); + bulkScan.set_id("test-bulk-scan"); + + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("example.com"); + scanTarget.setPort(443); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.TO_BE_EXECUTED); + + // Set up a latch to wait for job processing + CountDownLatch latch = new CountDownLatch(1); + persistenceProvider.setLatch(latch); + + // Start the worker + worker.start(); + + // Submit a job + orchestrationProvider.submitJob(jobDescription); + + // Wait for the job to be processed + assertTrue(latch.await(10, TimeUnit.SECONDS), "Job should be processed within 10 seconds"); + + // Verify the result was persisted + assertTrue(persistenceProvider.hasReceivedScanResult()); + } + + @Test + void testHandleScanJobWithTimeout() throws InterruptedException { + // Create a worker with a very short timeout + commandConfig = new WorkerCommandConfig(); + commandConfig.setParallelScanThreads(2); + commandConfig.setParallelConnectionThreads(4); + commandConfig.setScanTimeout(100); // Very short timeout + + worker = new TestWorker(commandConfig, orchestrationProvider, persistenceProvider); + worker.setUseTestBulkScanWorkerManager(true); + + // Create test data + TestScanConfig scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + BulkScan bulkScan = + new BulkScan( + this.getClass(), + this.getClass(), + "TestScan", + scanConfig, + System.currentTimeMillis(), + false, + null); + bulkScan.set_id("test-bulk-scan-timeout"); + + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("timeout.example.com"); + scanTarget.setPort(443); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.TO_BE_EXECUTED); + + // Set up a latch to wait for job processing + CountDownLatch latch = new CountDownLatch(1); + persistenceProvider.setLatch(latch); + TestBulkScanWorkerManager.setSimulateTimeout(true); + + // Start the worker + worker.start(); + + // Submit a job + orchestrationProvider.submitJob(jobDescription); + + // Wait for the job to be processed + assertTrue(latch.await(15, TimeUnit.SECONDS), "Job should be processed within 15 seconds"); + + // Verify the result was persisted with CANCELLED status + assertTrue(persistenceProvider.hasReceivedScanResult()); + assertEquals(JobStatus.CANCELLED, persistenceProvider.getLastJobStatus()); + } + + @Test + void testHandleScanJobWithExecutionException() throws InterruptedException { + worker.setUseTestBulkScanWorkerManager(true); + + // Create test data + TestScanConfig scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + BulkScan bulkScan = + new BulkScan( + this.getClass(), + this.getClass(), + "TestScan", + scanConfig, + System.currentTimeMillis(), + false, + null); + bulkScan.set_id("test-bulk-scan-exception"); + + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("error.example.com"); + scanTarget.setPort(443); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.TO_BE_EXECUTED); + + // Set up a latch to wait for job processing + CountDownLatch latch = new CountDownLatch(1); + persistenceProvider.setLatch(latch); + TestBulkScanWorkerManager.setSimulateException(true); + + // Start the worker + worker.start(); + + // Submit a job + orchestrationProvider.submitJob(jobDescription); + + // Wait for the job to be processed + assertTrue(latch.await(10, TimeUnit.SECONDS), "Job should be processed within 10 seconds"); + + // Verify the result was persisted with ERROR status + assertTrue(persistenceProvider.hasReceivedScanResult()); + assertEquals(JobStatus.ERROR, persistenceProvider.getLastJobStatus()); + } + + @Test + void testHandleScanJobWithNullResult() throws InterruptedException { + worker.setUseTestBulkScanWorkerManager(true); + + // Create test data + TestScanConfig scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + BulkScan bulkScan = + new BulkScan( + this.getClass(), + this.getClass(), + "TestScan", + scanConfig, + System.currentTimeMillis(), + false, + null); + bulkScan.set_id("test-bulk-scan-null"); + + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("null.example.com"); + scanTarget.setPort(443); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.TO_BE_EXECUTED); + + // Set up a latch to wait for job processing + CountDownLatch latch = new CountDownLatch(1); + persistenceProvider.setLatch(latch); + TestBulkScanWorkerManager.setSimulateNullResult(true); + + // Start the worker + worker.start(); + + // Submit a job + orchestrationProvider.submitJob(jobDescription); + + // Wait for the job to be processed + assertTrue(latch.await(10, TimeUnit.SECONDS), "Job should be processed within 10 seconds"); + + // Verify the EMPTY status for null result + assertTrue(persistenceProvider.hasReceivedScanResult()); + assertEquals(JobStatus.EMPTY, persistenceProvider.getLastJobStatus()); + } + + @Test + void testHandleScanJobWithPersistenceException() throws InterruptedException { + // Create test data + TestScanConfig scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + BulkScan bulkScan = + new BulkScan( + this.getClass(), + this.getClass(), + "TestScan", + scanConfig, + System.currentTimeMillis(), + false, + null); + bulkScan.set_id("test-bulk-scan-persist-error"); + + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("persist-error.example.com"); + scanTarget.setPort(443); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.TO_BE_EXECUTED); + + // Set up a latch to wait for job processing + CountDownLatch latch = new CountDownLatch(1); + persistenceProvider.setThrowException(true); + orchestrationProvider.setNotificationLatch(latch); + + // Start the worker + worker.start(); + + // Submit a job + orchestrationProvider.submitJob(jobDescription); + + // Wait for the job to be processed + assertTrue(latch.await(10, TimeUnit.SECONDS), "Job should be processed within 10 seconds"); + + // Verify notification was sent even though persistence failed + assertEquals( + JobStatus.INTERNAL_ERROR, orchestrationProvider.getLastNotifiedJob().getStatus()); + } + + @Test + void testHandleScanJobWithSecondTimeoutException() throws InterruptedException { + // Create a worker with a very short timeout + commandConfig = new WorkerCommandConfig(); + commandConfig.setParallelScanThreads(2); + commandConfig.setParallelConnectionThreads(4); + commandConfig.setScanTimeout(100); // Very short timeout + + worker = new TestWorker(commandConfig, orchestrationProvider, persistenceProvider); + worker.setUseTestBulkScanWorkerManager(true); + + // Create test data + TestScanConfig scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + BulkScan bulkScan = + new BulkScan( + this.getClass(), + this.getClass(), + "TestScan", + scanConfig, + System.currentTimeMillis(), + false, + null); + bulkScan.set_id("test-bulk-scan-double-timeout"); + + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("double-timeout.example.com"); + scanTarget.setPort(443); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.TO_BE_EXECUTED); + + // Set up a latch to wait for job processing + CountDownLatch latch = new CountDownLatch(1); + persistenceProvider.setLatch(latch); + + // Configure TestBulkScanWorkerManager to simulate double timeout + TestBulkScanWorkerManager.setSimulateTimeout(true); + TestBulkScanWorkerManager.setSimulateSecondTimeoutException(true); + + // Start the worker + worker.start(); + + // Submit a job + orchestrationProvider.submitJob(jobDescription); + + // Wait for the job to be processed + assertTrue(latch.await(20, TimeUnit.SECONDS), "Job should be processed within 20 seconds"); + + // Verify the job was marked as CANCELLED + assertTrue(persistenceProvider.hasReceivedScanResult()); + assertEquals(JobStatus.CANCELLED, persistenceProvider.getLastJobStatus()); + } + + @Test + void testHandleScanJobWithUnexpectedException() throws InterruptedException { + // This test will verify the catch-all exception handler + worker.setUseTestBulkScanWorkerManager(true); + + // Create test data + TestScanConfig scanConfig = new TestScanConfig(ScannerDetail.ALL, 1, 5000); + BulkScan bulkScan = + new BulkScan( + this.getClass(), + this.getClass(), + "TestScan", + scanConfig, + System.currentTimeMillis(), + false, + null); + bulkScan.set_id("test-bulk-scan-unexpected"); + + ScanTarget scanTarget = new ScanTarget(); + scanTarget.setHostname("unexpected.example.com"); + scanTarget.setPort(443); + + ScanJobDescription jobDescription = + new ScanJobDescription(scanTarget, bulkScan, JobStatus.TO_BE_EXECUTED); + + // Set up a latch to wait for job processing + CountDownLatch latch = new CountDownLatch(1); + persistenceProvider.setLatch(latch); + + // We'll test this by verifying the normal flow still works + TestBulkScanWorkerManager.reset(); + + // Start the worker + worker.start(); + + // Submit a job + orchestrationProvider.submitJob(jobDescription); + + // Wait for the job to be processed + assertTrue(latch.await(10, TimeUnit.SECONDS), "Job should be processed within 10 seconds"); + + // Verify the result was persisted + assertTrue(persistenceProvider.hasReceivedScanResult()); + } + + // Test implementation of IOrchestrationProvider + private static class TestOrchestrationProvider implements IOrchestrationProvider { + private ScanJobConsumer consumer; + private int prefetchCount; + private boolean simulateTimeout; + private boolean simulateException; + private boolean simulateNullResult; + private CountDownLatch notificationLatch; + private ScanJobDescription lastNotifiedJob; + + @Override + public void submitScanJob(ScanJobDescription scanJobDescription) {} + + @Override + public void registerScanJobConsumer(ScanJobConsumer scanJobConsumer, int prefetchCount) { + this.consumer = scanJobConsumer; + this.prefetchCount = prefetchCount; + } + + @Override + public void registerDoneNotificationConsumer( + BulkScan bulkScan, + de.rub.nds.crawler.orchestration.DoneNotificationConsumer + doneNotificationConsumer) {} + + @Override + public void notifyOfDoneScanJob(ScanJobDescription scanJobDescription) { + lastNotifiedJob = scanJobDescription; + if (notificationLatch != null) { + notificationLatch.countDown(); + } + } + + @Override + public void closeConnection() {} + + public boolean isConsumerRegistered() { + return consumer != null; + } + + public int getPrefetchCount() { + return prefetchCount; + } + + public void submitJob(ScanJobDescription job) { + if (consumer != null) { + consumer.consumeScanJob(job); + } + } + + public void setSimulateTimeout(boolean simulateTimeout) { + this.simulateTimeout = simulateTimeout; + } + + public void setSimulateException(boolean simulateException) { + this.simulateException = simulateException; + } + + public void setSimulateNullResult(boolean simulateNullResult) { + this.simulateNullResult = simulateNullResult; + } + + public void setNotificationLatch(CountDownLatch latch) { + this.notificationLatch = latch; + } + + public ScanJobDescription getLastNotifiedJob() { + return lastNotifiedJob; + } + } + + // Test implementation of IPersistenceProvider + private static class TestPersistenceProvider implements IPersistenceProvider { + private boolean receivedScanResult = false; + private CountDownLatch latch; + private boolean throwException = false; + private JobStatus lastJobStatus; + + @Override + public void insertScanResult(ScanResult scanResult, ScanJobDescription job) { + if (throwException) { + throw new RuntimeException("Simulated persistence exception"); + } + receivedScanResult = true; + lastJobStatus = job.getStatus(); + if (latch != null) { + latch.countDown(); + } + } + + @Override + public void insertBulkScan(BulkScan bulkScan) {} + + @Override + public void updateBulkScan(BulkScan bulkScan) {} + + public boolean hasReceivedScanResult() { + return receivedScanResult; + } + + public void setLatch(CountDownLatch latch) { + this.latch = latch; + } + + public void setThrowException(boolean throwException) { + this.throwException = throwException; + } + + public JobStatus getLastJobStatus() { + return lastJobStatus; + } + } +} diff --git a/src/test/java/de/rub/nds/crawler/test/TestScanConfig.java b/src/test/java/de/rub/nds/crawler/test/TestScanConfig.java new file mode 100644 index 0000000..1af2eb1 --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/test/TestScanConfig.java @@ -0,0 +1,53 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2023 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.test; + +import de.rub.nds.crawler.core.BulkScanWorker; +import de.rub.nds.crawler.data.ScanConfig; +import de.rub.nds.crawler.data.ScanTarget; +import de.rub.nds.scanner.core.config.ScannerDetail; +import org.bson.Document; + +public class TestScanConfig extends ScanConfig { + + public TestScanConfig(ScannerDetail scannerDetail, int reexecutions, int timeout) { + super(scannerDetail, reexecutions, timeout); + } + + @Override + public BulkScanWorker createWorker( + String bulkScanID, int parallelConnectionThreads, int parallelScanThreads) { + return new TestBulkScanWorker(bulkScanID); + } + + private static class TestBulkScanWorker extends BulkScanWorker { + public TestBulkScanWorker(String bulkScanId) { + super(bulkScanId, new TestScanConfig(ScannerDetail.ALL, 1, 5000), 4); + } + + @Override + public Document scan(ScanTarget scanTarget) { + Document doc = new Document(); + doc.put("test", true); + doc.put("hostname", scanTarget.getHostname()); + doc.put("port", scanTarget.getPort()); + return doc; + } + + @Override + protected void initInternal() { + // No initialization needed for test + } + + @Override + protected void cleanupInternal() { + // No cleanup needed for test + } + } +}