diff --git a/src/main/java/de/rub/nds/crawler/data/BulkScan.java b/src/main/java/de/rub/nds/crawler/data/BulkScan.java index 980c089..a1703df 100644 --- a/src/main/java/de/rub/nds/crawler/data/BulkScan.java +++ b/src/main/java/de/rub/nds/crawler/data/BulkScan.java @@ -56,6 +56,17 @@ public class BulkScan implements Serializable { @SuppressWarnings("unused") private BulkScan() {} + /** + * Creates a new bulk scan with the specified configuration. + * + * @param scannerClass the scanner class used for scanning + * @param crawlerClass the crawler class used for crawling + * @param name the name of the bulk scan + * @param scanConfig the scan configuration + * @param startTime the start time in milliseconds since epoch + * @param monitored whether the scan should be monitored + * @param notifyUrl the URL to notify when scan completes + */ public BulkScan( Class scannerClass, Class crawlerClass, @@ -76,140 +87,310 @@ public BulkScan( this.notifyUrl = notifyUrl; } + /** + * Gets the unique identifier of the bulk scan. + * + * @return the bulk scan ID + */ // Getter naming important for correct serialization, do not change! public String get_id() { return _id; } + /** + * Gets the name of the bulk scan. + * + * @return the scan name + */ public String getName() { return this.name; } + /** + * Gets the collection name for this bulk scan. + * + * @return the collection name + */ public String getCollectionName() { return this.collectionName; } + /** + * Gets the scan configuration. + * + * @return the scan configuration + */ public ScanConfig getScanConfig() { return this.scanConfig; } + /** + * Checks if the bulk scan is monitored. + * + * @return true if monitored, false otherwise + */ public boolean isMonitored() { return this.monitored; } + /** + * Checks if the bulk scan has finished. + * + * @return true if finished, false otherwise + */ public boolean isFinished() { return this.finished; } + /** + * Gets the start time of the bulk scan. + * + * @return the start time in milliseconds since epoch + */ public long getStartTime() { return this.startTime; } + /** + * Gets the end time of the bulk scan. + * + * @return the end time in milliseconds since epoch + */ public long getEndTime() { return this.endTime; } + /** + * Gets the number of targets given for scanning. + * + * @return the number of targets + */ public int getTargetsGiven() { return this.targetsGiven; } + /** + * Gets the number of scan jobs published. + * + * @return the number of published scan jobs + */ public long getScanJobsPublished() { return this.scanJobsPublished; } + /** + * Gets the number of successful scans. + * + * @return the number of successful scans + */ public int getSuccessfulScans() { return this.successfulScans; } + /** + * Gets the notification URL. + * + * @return the URL to notify when scan completes + */ public String getNotifyUrl() { return this.notifyUrl; } + /** + * Gets the scanner version used for this bulk scan. + * + * @return the scanner version + */ public String getScannerVersion() { return this.scannerVersion; } + /** + * Gets the crawler version used for this bulk scan. + * + * @return the crawler version + */ public String getCrawlerVersion() { return this.crawlerVersion; } + /** + * Sets the unique identifier of the bulk scan. + * + * @param _id the bulk scan ID + */ // Setter naming important for correct serialization, do not change! public void set_id(String _id) { this._id = _id; } + /** + * Sets the name of the bulk scan. + * + * @param name the scan name + */ public void setName(String name) { this.name = name; } + /** + * Sets the collection name for this bulk scan. + * + * @param collectionName the collection name + */ public void setCollectionName(String collectionName) { this.collectionName = collectionName; } + /** + * Sets the scan configuration. + * + * @param scanConfig the scan configuration + */ public void setScanConfig(ScanConfig scanConfig) { this.scanConfig = scanConfig; } + /** + * Sets whether the bulk scan is monitored. + * + * @param monitored true to enable monitoring, false otherwise + */ public void setMonitored(boolean monitored) { this.monitored = monitored; } + /** + * Sets whether the bulk scan has finished. + * + * @param finished true if finished, false otherwise + */ public void setFinished(boolean finished) { this.finished = finished; } + /** + * Sets the start time of the bulk scan. + * + * @param startTime the start time in milliseconds since epoch + */ public void setStartTime(long startTime) { this.startTime = startTime; } + /** + * Sets the end time of the bulk scan. + * + * @param endTime the end time in milliseconds since epoch + */ public void setEndTime(long endTime) { this.endTime = endTime; } + /** + * Sets the number of targets given for scanning. + * + * @param targetsGiven the number of targets + */ public void setTargetsGiven(int targetsGiven) { this.targetsGiven = targetsGiven; } + /** + * Sets the number of scan jobs published. + * + * @param scanJobsPublished the number of published scan jobs + */ public void setScanJobsPublished(long scanJobsPublished) { this.scanJobsPublished = scanJobsPublished; } + /** + * Sets the number of successful scans. + * + * @param successfulScans the number of successful scans + */ public void setSuccessfulScans(int successfulScans) { this.successfulScans = successfulScans; } + /** + * Sets the notification URL. + * + * @param notifyUrl the URL to notify when scan completes + */ public void setNotifyUrl(String notifyUrl) { this.notifyUrl = notifyUrl; } + /** + * Sets the scanner version used for this bulk scan. + * + * @param scannerVersion the scanner version + */ public void setScannerVersion(String scannerVersion) { this.scannerVersion = scannerVersion; } + /** + * Sets the crawler version used for this bulk scan. + * + * @param crawlerVersion the crawler version + */ public void setCrawlerVersion(String crawlerVersion) { this.crawlerVersion = crawlerVersion; } + /** + * Gets the map of job status counters. + * + * @return the job status counters map + */ public Map getJobStatusCounters() { return jobStatusCounters; } + /** + * Sets the map of job status counters. + * + * @param jobStatusCounters the job status counters map + */ public void setJobStatusCounters(Map jobStatusCounters) { this.jobStatusCounters = jobStatusCounters; } + /** + * Gets the number of scan jobs with resolution errors. + * + * @return the number of scan jobs with resolution errors + */ public long getScanJobsResolutionErrors() { return scanJobsResolutionErrors; } + /** + * Sets the number of scan jobs with resolution errors. + * + * @param scanJobsResolutionErrors the number of scan jobs with resolution errors + */ public void setScanJobsResolutionErrors(long scanJobsResolutionErrors) { this.scanJobsResolutionErrors = scanJobsResolutionErrors; } + /** + * Gets the number of scan jobs that were denylisted. + * + * @return the number of denylisted scan jobs + */ public long getScanJobsDenylisted() { return scanJobsDenylisted; } + /** + * Sets the number of scan jobs that were denylisted. + * + * @param scanJobsDenylisted the number of denylisted scan jobs + */ public void setScanJobsDenylisted(long scanJobsDenylisted) { this.scanJobsDenylisted = scanJobsDenylisted; } diff --git a/src/main/java/de/rub/nds/crawler/data/BulkScanInfo.java b/src/main/java/de/rub/nds/crawler/data/BulkScanInfo.java index 1e40e41..1a4dc8c 100644 --- a/src/main/java/de/rub/nds/crawler/data/BulkScanInfo.java +++ b/src/main/java/de/rub/nds/crawler/data/BulkScanInfo.java @@ -21,24 +21,51 @@ public class BulkScanInfo implements Serializable { private final boolean isMonitored; + /** + * Creates a new BulkScanInfo from the given bulk scan. + * + * @param bulkScan the bulk scan to extract information from + */ public BulkScanInfo(BulkScan bulkScan) { this.bulkScanId = bulkScan.get_id(); this.scanConfig = bulkScan.getScanConfig(); this.isMonitored = bulkScan.isMonitored(); } + /** + * Gets the bulk scan ID. + * + * @return the bulk scan ID + */ public String getBulkScanId() { return bulkScanId; } + /** + * Gets the scan configuration. + * + * @return the scan configuration + */ public ScanConfig getScanConfig() { return scanConfig; } + /** + * Gets the scan configuration cast to the specified type. + * + * @param clazz the class to cast the scan configuration to + * @param the type to cast to, must extend ScanConfig + * @return the scan configuration cast to the specified type + */ public T getScanConfig(Class clazz) { return clazz.cast(scanConfig); } + /** + * Checks if the bulk scan is monitored. + * + * @return true if monitored, false otherwise + */ public boolean isMonitored() { return isMonitored; } diff --git a/src/main/java/de/rub/nds/crawler/data/BulkScanJobCounters.java b/src/main/java/de/rub/nds/crawler/data/BulkScanJobCounters.java index bfaac3a..6d8c147 100644 --- a/src/main/java/de/rub/nds/crawler/data/BulkScanJobCounters.java +++ b/src/main/java/de/rub/nds/crawler/data/BulkScanJobCounters.java @@ -20,6 +20,11 @@ public class BulkScanJobCounters { private final AtomicInteger totalJobDoneCount = new AtomicInteger(0); private final Map jobStatusCounters = new EnumMap<>(JobStatus.class); + /** + * Creates a new BulkScanJobCounters for the given bulk scan. + * + * @param bulkScan the bulk scan to track job counters for + */ public BulkScanJobCounters(BulkScan bulkScan) { this.bulkScan = bulkScan; for (JobStatus jobStatus : JobStatus.values()) { @@ -30,10 +35,20 @@ public BulkScanJobCounters(BulkScan bulkScan) { } } + /** + * Gets the bulk scan associated with these counters. + * + * @return the bulk scan + */ public BulkScan getBulkScan() { return bulkScan; } + /** + * Gets a copy of the job status counters map. + * + * @return a copy of the job status counters + */ public Map getJobStatusCountersCopy() { EnumMap ret = new EnumMap<>(JobStatus.class); for (Map.Entry entry : jobStatusCounters.entrySet()) { @@ -42,10 +57,22 @@ public Map getJobStatusCountersCopy() { return ret; } + /** + * Gets the count for a specific job status. + * + * @param jobStatus the job status to get the count for + * @return the count for the specified job status + */ public int getJobStatusCount(JobStatus jobStatus) { return jobStatusCounters.get(jobStatus).get(); } + /** + * Increments the count for a specific job status. + * + * @param jobStatus the job status to increment the count for + * @return the new total job done count + */ public int increaseJobStatusCount(JobStatus jobStatus) { jobStatusCounters.get(jobStatus).incrementAndGet(); return totalJobDoneCount.incrementAndGet(); diff --git a/src/main/java/de/rub/nds/crawler/data/ScanConfig.java b/src/main/java/de/rub/nds/crawler/data/ScanConfig.java index 8f91fc2..1a8ed57 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanConfig.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanConfig.java @@ -29,30 +29,68 @@ protected ScanConfig(ScannerDetail scannerDetail, int reexecutions, int timeout) this.timeout = timeout; } + /** + * Gets the scanner detail configuration. + * + * @return the scanner detail + */ public ScannerDetail getScannerDetail() { return this.scannerDetail; } + /** + * Gets the number of re-executions configured. + * + * @return the number of re-executions + */ public int getReexecutions() { return this.reexecutions; } + /** + * Gets the timeout value in seconds. + * + * @return the timeout value + */ public int getTimeout() { return this.timeout; } + /** + * Sets the scanner detail configuration. + * + * @param scannerDetail the scanner detail to set + */ public void setScannerDetail(ScannerDetail scannerDetail) { this.scannerDetail = scannerDetail; } + /** + * Sets the number of re-executions. + * + * @param reexecutions the number of re-executions to set + */ public void setReexecutions(int reexecutions) { this.reexecutions = reexecutions; } + /** + * Sets the timeout value in seconds. + * + * @param timeout the timeout value to set + */ public void setTimeout(int timeout) { this.timeout = timeout; } + /** + * Creates a worker for bulk scanning with the specified configuration. + * + * @param bulkScanID the bulk scan ID + * @param parallelConnectionThreads the number of parallel connection threads + * @param parallelScanThreads the number of parallel scan threads + * @return a new bulk scan worker + */ public abstract BulkScanWorker createWorker( String bulkScanID, int parallelConnectionThreads, int parallelScanThreads); } diff --git a/src/main/java/de/rub/nds/crawler/data/ScanJobDescription.java b/src/main/java/de/rub/nds/crawler/data/ScanJobDescription.java index 841b410..edd470f 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanJobDescription.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanJobDescription.java @@ -30,6 +30,15 @@ public class ScanJobDescription implements Serializable { private final String collectionName; + /** + * Creates a new scan job description with the specified parameters. + * + * @param scanTarget the target to scan + * @param bulkScanInfo the bulk scan information + * @param dbName the database name for storing results + * @param collectionName the collection name for storing results + * @param status the initial job status + */ public ScanJobDescription( ScanTarget scanTarget, BulkScanInfo bulkScanInfo, @@ -43,6 +52,13 @@ public ScanJobDescription( this.status = status; } + /** + * Creates a new scan job description from a bulk scan. + * + * @param scanTarget the target to scan + * @param bulkScan the bulk scan to extract information from + * @param status the initial job status + */ public ScanJobDescription(ScanTarget scanTarget, BulkScan bulkScan, JobStatus status) { this( scanTarget, @@ -59,30 +75,66 @@ private void readObject(java.io.ObjectInputStream in) deliveryTag = Optional.empty(); } + /** + * Gets the scan target. + * + * @return the scan target + */ public ScanTarget getScanTarget() { return scanTarget; } + /** + * Gets the database name for storing results. + * + * @return the database name + */ public String getDbName() { return dbName; } + /** + * Gets the collection name for storing results. + * + * @return the collection name + */ public String getCollectionName() { return collectionName; } + /** + * Gets the current job status. + * + * @return the job status + */ public JobStatus getStatus() { return status; } + /** + * Sets the job status. + * + * @param status the new job status + */ public void setStatus(JobStatus status) { this.status = status; } + /** + * Gets the delivery tag for message acknowledgement. + * + * @return the delivery tag + */ public long getDeliveryTag() { return deliveryTag.get(); } + /** + * Sets the delivery tag for message acknowledgement. + * + * @param deliveryTag the delivery tag to set + * @throws IllegalStateException if the delivery tag was already set + */ public void setDeliveryTag(Long deliveryTag) { if (this.deliveryTag.isPresent()) { throw new IllegalStateException("Delivery tag already set"); @@ -90,6 +142,11 @@ public void setDeliveryTag(Long deliveryTag) { this.deliveryTag = Optional.of(deliveryTag); } + /** + * Gets the bulk scan information. + * + * @return the bulk scan information + */ public BulkScanInfo getBulkScanInfo() { return bulkScanInfo; } diff --git a/src/main/java/de/rub/nds/crawler/data/ScanResult.java b/src/main/java/de/rub/nds/crawler/data/ScanResult.java index ebd5de5..6bd2306 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanResult.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanResult.java @@ -35,6 +35,13 @@ private ScanResult( this.result = result; } + /** + * Creates a new scan result from a scan job description. + * + * @param scanJobDescription the scan job description + * @param result the scan result document + * @throws IllegalArgumentException if the scan job description is in TO_BE_EXECUTED state + */ public ScanResult(ScanJobDescription scanJobDescription, Document result) { this( scanJobDescription.getBulkScanInfo().getBulkScanId(), @@ -47,6 +54,14 @@ public ScanResult(ScanJobDescription scanJobDescription, Document result) { } } + /** + * Creates a scan result from an exception. + * + * @param scanJobDescription the scan job description + * @param e the exception that occurred + * @return a new scan result containing the exception + * @throws IllegalArgumentException if the scan job description is not in an error state + */ public static ScanResult fromException(ScanJobDescription scanJobDescription, Exception e) { if (!scanJobDescription.getStatus().isError()) { throw new IllegalArgumentException("ScanJobDescription must be in an error state"); @@ -56,28 +71,58 @@ public static ScanResult fromException(ScanJobDescription scanJobDescription, Ex return new ScanResult(scanJobDescription, errorDocument); } + /** + * Gets the unique identifier of the scan result. + * + * @return the scan result ID + */ @JsonProperty("_id") public String getId() { return this.id; } + /** + * Sets the unique identifier of the scan result. + * + * @param id the scan result ID + */ @JsonProperty("_id") public void setId(String id) { this.id = id; } + /** + * Gets the bulk scan ID associated with this result. + * + * @return the bulk scan ID + */ public String getBulkScan() { return this.bulkScan; } + /** + * Gets the scan target for this result. + * + * @return the scan target + */ public ScanTarget getScanTarget() { return this.scanTarget; } + /** + * Gets the result document containing scan data. + * + * @return the result document + */ public Document getResult() { return this.result; } + /** + * Gets the job status of this scan result. + * + * @return the job status + */ public JobStatus getResultStatus() { return jobStatus; } diff --git a/src/main/java/de/rub/nds/crawler/data/ScanTarget.java b/src/main/java/de/rub/nds/crawler/data/ScanTarget.java index b5299b6..95545ea 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanTarget.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanTarget.java @@ -99,41 +99,87 @@ public static Pair fromTargetString( private int trancoRank; + /** Creates a new scan target with default values. */ public ScanTarget() {} + /** + * Returns a string representation of the scan target. + * + * @return the hostname if available, otherwise the IP address + */ @Override public String toString() { return hostname != null ? hostname : ip; } + /** + * Gets the IP address of the scan target. + * + * @return the IP address + */ public String getIp() { return this.ip; } + /** + * Gets the hostname of the scan target. + * + * @return the hostname + */ public String getHostname() { return this.hostname; } + /** + * Gets the port number of the scan target. + * + * @return the port number + */ public int getPort() { return this.port; } + /** + * Gets the Tranco rank of the scan target. + * + * @return the Tranco rank + */ public int getTrancoRank() { return this.trancoRank; } + /** + * Sets the IP address of the scan target. + * + * @param ip the IP address to set + */ public void setIp(String ip) { this.ip = ip; } + /** + * Sets the hostname of the scan target. + * + * @param hostname the hostname to set + */ public void setHostname(String hostname) { this.hostname = hostname; } + /** + * Sets the port number of the scan target. + * + * @param port the port number to set + */ public void setPort(int port) { this.port = port; } + /** + * Sets the Tranco rank of the scan target. + * + * @param trancoRank the Tranco rank to set + */ public void setTrancoRank(int trancoRank) { this.trancoRank = trancoRank; }