feat: Support multiple IPs per hostname in ScanTarget

src/main/java/de/rub/nds/crawler/data/ScanTarget.java

@@ -13,6 +13,9 @@
 import java.io.Serializable;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.validator.routines.InetAddressValidator;
 import org.apache.logging.log4j.LogManager;
@@ -67,12 +70,21 @@ public static Pair<ScanTarget, JobStatus> fromTargetString(
 
         if (InetAddressValidator.getInstance().isValid(targetString)) {
             target.setIp(targetString);
+            target.setIps(Collections.singletonList(targetString));
         } else {
             target.setHostname(targetString);
             try {
-                // TODO this only allows one IP per hostname; it may be interesting to scan all IPs
-                // for a domain, or at least one v4 and one v6
-                target.setIp(InetAddress.getByName(targetString).getHostAddress());
+                // Resolve all IP addresses for the hostname
+                InetAddress[] addresses = InetAddress.getAllByName(targetString);
+                List<String> resolvedIps = new ArrayList<>();
+                for (InetAddress address : addresses) {
+                    resolvedIps.add(address.getHostAddress());
+                }
+                target.setIps(resolvedIps);
+                // Set the first IP for backward compatibility
+                if (!resolvedIps.isEmpty()) {
+                    target.setIp(resolvedIps.get(0));
+                }
             } catch (UnknownHostException e) {
                 LOGGER.error(
                         "Host {} is unknown or can not be reached with error {}.", targetString, e);
@@ -93,23 +105,47 @@ public static Pair<ScanTarget, JobStatus> fromTargetString(
 
     private String ip;
 
+    private List<String> ips;
+
     private String hostname;
 
     private int port;
 
     private int trancoRank;
 
-    public ScanTarget() {}
+    public ScanTarget() {
+        this.ips = new ArrayList<>();
+    }
 
     @Override
     public String toString() {
-        return hostname != null ? hostname : ip;
+        if (hostname != null) {
+            return hostname;
+        } else if (!ips.isEmpty()) {
+            // If multiple IPs, show all
+            if (ips.size() > 1) {
+                return "[" + String.join(", ", ips) + "]";
+            } else {
+                return ips.get(0);
+            }
+        } else {
+            return ip;
+        }
     }
 
+    /**
+     * @deprecated Use {@link #getIps()} instead to get all IP addresses. This method returns only
+     *     the first IP for backward compatibility.
+     */
+    @Deprecated
     public String getIp() {
         return this.ip;
     }
 
+    public List<String> getIps() {
+        return new ArrayList<>(this.ips);
+    }
+
     public String getHostname() {
         return this.hostname;
     }
@@ -122,8 +158,25 @@ public int getTrancoRank() {
         return this.trancoRank;
     }
 
+    /**
+     * @deprecated Use {@link #setIps(List)} instead to set all IP addresses. This method is kept
+     *     for backward compatibility.
+     */
+    @Deprecated
     public void setIp(String ip) {
         this.ip = ip;
+        if (this.ips.isEmpty() || !this.ips.get(0).equals(ip)) {
+            this.ips = new ArrayList<>();
+            this.ips.add(ip);
+        }
+    }
+
+    public void setIps(List<String> ips) {
+        this.ips = new ArrayList<>(ips);
+        // Set the first IP for backward compatibility
+        if (!ips.isEmpty()) {
+            this.ip = ips.get(0);
+        }
     }
 
     public void setHostname(String hostname) {

src/main/java/de/rub/nds/crawler/denylist/DenylistFileProvider.java

@@ -69,9 +69,19 @@ private boolean isInSubnet(String ip, SubnetUtils.SubnetInfo subnetInfo) {
 
     @Override
     public synchronized boolean isDenylisted(ScanTarget target) {
-        return domainDenylistSet.contains(target.getHostname())
-                || ipDenylistSet.contains(target.getIp())
-                || cidrDenylist.stream()
-                        .anyMatch(subnetInfo -> isInSubnet(target.getIp(), subnetInfo));
+        // Check if hostname is denylisted
+        if (domainDenylistSet.contains(target.getHostname())) {
+            return true;
+        }
+
+        // Check if any of the IPs are denylisted
+        for (String ip : target.getIps()) {
+            if (ipDenylistSet.contains(ip)
+                    || cidrDenylist.stream().anyMatch(subnetInfo -> isInSubnet(ip, subnetInfo))) {
+                return true;
+            }
+        }
+
+        return false;
     }
 }

src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java

@@ -192,7 +192,9 @@ private JacksonMongoCollection<ScanResult> initResultCollection(
                                 ScanResult.class,
                                 UuidRepresentation.STANDARD);
         // createIndex is idempotent, hence we do not need to check if an index exists
-        collection.createIndex(Indexes.ascending("scanTarget.ip"));
+        collection.createIndex(
+                Indexes.ascending("scanTarget.ip")); // Keep for backward compatibility
+        collection.createIndex(Indexes.ascending("scanTarget.ips")); // Index for multiple IPs
         collection.createIndex(Indexes.ascending("scanTarget.hostname"));
         collection.createIndex(Indexes.ascending("scanTarget.trancoRank"));
         collection.createIndex(Indexes.ascending("scanTarget.resultStatus"));

src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java

@@ -0,0 +1,134 @@
+/*
+ * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner
+ *
+ * Copyright 2018-2022 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH
+ *
+ * Licensed under Apache License, Version 2.0
+ * http://www.apache.org/licenses/LICENSE-2.0.txt
+ */
+package de.rub.nds.crawler.data;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import de.rub.nds.crawler.constant.JobStatus;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.lang3.tuple.Pair;
+import org.junit.jupiter.api.Test;
+
+class ScanTargetTest {
+
+    @Test
+    void testFromTargetStringWithHostname() {
+        // Test hostname resolution to multiple IPs
+        Pair<ScanTarget, JobStatus> result = ScanTarget.fromTargetString("localhost", 443, null);
+
+        assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight());
+        ScanTarget target = result.getLeft();
+
+        assertEquals("localhost", target.getHostname());
+        assertEquals(443, target.getPort());
+        assertNotNull(target.getIps());
+        assertFalse(target.getIps().isEmpty());
+        // localhost should resolve to at least one IP
+        assertTrue(target.getIps().size() >= 1);
+        // The deprecated getIp() should return the first IP
+        assertEquals(target.getIps().get(0), target.getIp());
+    }
+
+    @Test
+    void testFromTargetStringWithIpAddress() {
+        Pair<ScanTarget, JobStatus> result = ScanTarget.fromTargetString("127.0.0.1", 443, null);
+
+        assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight());
+        ScanTarget target = result.getLeft();
+
+        assertNull(target.getHostname());
+        assertEquals("127.0.0.1", target.getIp());
+        assertEquals(Arrays.asList("127.0.0.1"), target.getIps());
+        assertEquals(443, target.getPort());
+    }
+
+    @Test
+    void testFromTargetStringWithPort() {
+        Pair<ScanTarget, JobStatus> result =
+                ScanTarget.fromTargetString("127.0.0.1:8443", 443, null);
+
+        assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight());
+        ScanTarget target = result.getLeft();
+
+        assertEquals("127.0.0.1", target.getIp());
+        assertEquals(Arrays.asList("127.0.0.1"), target.getIps());
+        assertEquals(8443, target.getPort());
+    }
+
+    @Test
+    void testFromTargetStringWithTrancoRank() {
+        Pair<ScanTarget, JobStatus> result =
+                ScanTarget.fromTargetString("100,127.0.0.1", 443, null);
+
+        assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight());
+        ScanTarget target = result.getLeft();
+
+        assertEquals(100, target.getTrancoRank());
+        assertEquals("127.0.0.1", target.getIp());
+        assertEquals(Arrays.asList("127.0.0.1"), target.getIps());
+    }
+
+    @Test
+    void testFromTargetStringUnresolvableHost() {
+        Pair<ScanTarget, JobStatus> result =
+                ScanTarget.fromTargetString("this-host-should-not-exist-12345.invalid", 443, null);
+
+        assertEquals(JobStatus.UNRESOLVABLE, result.getRight());
+    }
+
+    @Test
+    void testSetIpsAndBackwardCompatibility() {
+        ScanTarget target = new ScanTarget();
+        List<String> ips = Arrays.asList("192.168.1.1", "192.168.1.2", "192.168.1.3");
+
+        target.setIps(ips);
+
+        // Check that all IPs are stored
+        assertEquals(ips, target.getIps());
+        // Check backward compatibility - getIp() should return the first IP
+        assertEquals("192.168.1.1", target.getIp());
+    }
+
+    @Test
+    void testSetIpBackwardCompatibility() {
+        ScanTarget target = new ScanTarget();
+
+        // Test deprecated setIp method
+        target.setIp("10.0.0.1");
+
+        assertEquals("10.0.0.1", target.getIp());
+        assertEquals(Arrays.asList("10.0.0.1"), target.getIps());
+
+        // Setting another IP should update the list
+        target.setIp("10.0.0.2");
+        assertEquals("10.0.0.2", target.getIp());
+        assertEquals(Arrays.asList("10.0.0.2"), target.getIps());
+    }
+
+    @Test
+    void testToStringWithMultipleIps() {
+        ScanTarget target = new ScanTarget();
+
+        // Test with hostname
+        target.setHostname("example.com");
+        target.setIps(Arrays.asList("192.168.1.1", "192.168.1.2"));
+        assertEquals("example.com", target.toString());
+
+        // Test with single IP (no hostname)
+        target = new ScanTarget();
+        target.setIps(Arrays.asList("192.168.1.1"));
+        assertEquals("192.168.1.1", target.toString());
+
+        // Test with multiple IPs (no hostname)
+        target = new ScanTarget();
+        target.setIps(Arrays.asList("192.168.1.1", "192.168.1.2", "192.168.1.3"));
+        assertEquals("[192.168.1.1, 192.168.1.2, 192.168.1.3]", target.toString());
+    }
+}