Handle IPv6 only domains failing to be submitted.

While at that, also add more error handling.
Also add the possibility of comments in the TargetFileProvider.

Author: XoMEX

src/main/java/de/rub/nds/crawler/constant/JobStatus.java

@@ -13,6 +13,8 @@ public enum JobStatus {
     TO_BE_EXECUTED(false),
     /** The domain was not resolvable. An empty result was written to DB. */
     UNRESOLVABLE(true),
+    /** An uncaught exception occurred while resolving the host. */
+    RESOLUTION_ERROR(true),
     /** The domain was denylisted. An empty result was written to DB. */
     DENYLISTED(true),
     /** Job was successfully executed. Result was written to db. */

src/main/java/de/rub/nds/crawler/core/jobs/PublishBulkScanJob.java

@@ -76,17 +76,18 @@ public void execute(JobExecutionContext context) throws JobExecutionException {
                                             Function.identity(), Collectors.counting()));
 
             long submittedJobs = parsedJobStatuses.getOrDefault(JobStatus.TO_BE_EXECUTED, 0L);
+            long unresolvedJobs = parsedJobStatuses.getOrDefault(JobStatus.UNRESOLVABLE, 0L);
+            long denylistedJobs = parsedJobStatuses.getOrDefault(JobStatus.DENYLISTED, 0L);
+            long resolutionErrorJobs = parsedJobStatuses.getOrDefault(JobStatus.RESOLUTION_ERROR, 0L);
             bulkScan.setScanJobsPublished(submittedJobs);
-            bulkScan.setScanJobsResolutionErrors(
-                    parsedJobStatuses.getOrDefault(JobStatus.UNRESOLVABLE, 0L));
-            bulkScan.setScanJobsDenylisted(
-                    parsedJobStatuses.getOrDefault(JobStatus.DENYLISTED, 0L));
+            bulkScan.setScanJobsResolutionErrors(unresolvedJobs+resolutionErrorJobs);
+            bulkScan.setScanJobsDenylisted(denylistedJobs);
             persistenceProvider.updateBulkScan(bulkScan);
 
             if (controllerConfig.isMonitored() && submittedJobs == 0) {
                 progressMonitor.stopMonitoringAndFinalizeBulkScan(bulkScan.get_id());
             }
-            LOGGER.info("Submitted {} scan jobs to RabbitMq", submittedJobs);
+            LOGGER.info("Submitted {} scan jobs to RabbitMq (Not submitted: {} Unresolvable, {} Denylisted, {} unhandled Error)", submittedJobs,unresolvedJobs, denylistedJobs, resolutionErrorJobs);
         } catch (Exception e) {
             LOGGER.error("Exception while publishing BulkScan: ", e);
             JobExecutionException e2 = new JobExecutionException(e);
@@ -117,15 +118,26 @@ public JobSubmitter(
 
         @Override
         public JobStatus apply(String targetString) {
-            var targetInfo =
-                    ScanTarget.fromTargetString(targetString, defaultPort, denylistProvider);
-            ScanJobDescription jobDescription =
-                    new ScanJobDescription(targetInfo.getLeft(), bulkScan, targetInfo.getRight());
+            ScanJobDescription jobDescription;
+            ScanResult errorResult = null;
+            try {
+                var targetInfo =
+                        ScanTarget.fromTargetString(targetString, defaultPort, denylistProvider);
+                jobDescription =
+                        new ScanJobDescription(targetInfo.getLeft(), bulkScan, targetInfo.getRight());
+            }catch (Exception e){
+                jobDescription = new ScanJobDescription(new ScanTarget(), bulkScan, JobStatus.RESOLUTION_ERROR);
+                errorResult = ScanResult.fromException(jobDescription, e);
+                LOGGER.error("Error while creating ScanJobDescription for target '{}'", targetString, e);
+            }
+
             if (jobDescription.getStatus() == JobStatus.TO_BE_EXECUTED) {
                 orchestrationProvider.submitScanJob(jobDescription);
             } else {
-                persistenceProvider.insertScanResult(
-                        new ScanResult(jobDescription, null), jobDescription);
+                if(errorResult == null){
+                    errorResult = new ScanResult(jobDescription, null);
+                }
+                persistenceProvider.insertScanResult(errorResult, jobDescription);
             }
             return jobDescription.getStatus();
         }

src/main/java/de/rub/nds/crawler/denylist/DenylistFileProvider.java

@@ -58,11 +58,20 @@ public DenylistFileProvider(String denylistFilename) {
         }
     }
 
+    private boolean isInSubnet(String ip, SubnetUtils.SubnetInfo subnetInfo) {
+        try {
+            return subnetInfo.isInRange(ip);
+        } catch (IllegalArgumentException e) {
+            // most likely we tried to check an IPv6 address against an IPv4 subnet
+            return false;
+        }
+    }
+
     @Override
     public synchronized boolean isDenylisted(ScanTarget target) {
         return domainDenylistSet.contains(target.getHostname())
                 || ipDenylistSet.contains(target.getIp())
                 || cidrDenylist.stream()
-                        .anyMatch(subnetInfo -> subnetInfo.isInRange(target.getIp()));
+                        .anyMatch(subnetInfo -> isInSubnet(target.getIp(), subnetInfo));
     }
 }

src/main/java/de/rub/nds/crawler/targetlist/TargetFileProvider.java

@@ -32,7 +32,9 @@ public List<String> getTargetList() {
         LOGGER.info("Reading hostName list");
         List<String> targetList;
         try (Stream<String> lines = Files.lines(Paths.get(filename))) {
-            targetList = lines.collect(Collectors.toList());
+            // remove comments and empty lines
+            targetList = lines.filter(line -> !(line.startsWith("#") || line.isEmpty()))
+                    .collect(Collectors.toList());
         } catch (IOException ex) {
             throw new RuntimeException("Could not load " + filename, ex);
         }