diff --git a/src/main/java/de/rub/nds/crawler/data/ScanTarget.java b/src/main/java/de/rub/nds/crawler/data/ScanTarget.java index b5299b6..bc4bebc 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanTarget.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanTarget.java @@ -13,6 +13,9 @@ import java.io.Serializable; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.logging.log4j.LogManager; @@ -67,12 +70,21 @@ public static Pair fromTargetString( if (InetAddressValidator.getInstance().isValid(targetString)) { target.setIp(targetString); + target.setIps(Collections.singletonList(targetString)); } else { target.setHostname(targetString); try { - // TODO this only allows one IP per hostname; it may be interesting to scan all IPs - // for a domain, or at least one v4 and one v6 - target.setIp(InetAddress.getByName(targetString).getHostAddress()); + // Resolve all IP addresses for the hostname + InetAddress[] addresses = InetAddress.getAllByName(targetString); + List resolvedIps = new ArrayList<>(); + for (InetAddress address : addresses) { + resolvedIps.add(address.getHostAddress()); + } + target.setIps(resolvedIps); + // Set the first IP for backward compatibility + if (!resolvedIps.isEmpty()) { + target.setIp(resolvedIps.get(0)); + } } catch (UnknownHostException e) { LOGGER.error( "Host {} is unknown or can not be reached with error {}.", targetString, e); @@ -93,23 +105,47 @@ public static Pair fromTargetString( private String ip; + private List ips; + private String hostname; private int port; private int trancoRank; - public ScanTarget() {} + public ScanTarget() { + this.ips = new ArrayList<>(); + } @Override public String toString() { - return hostname != null ? hostname : ip; + if (hostname != null) { + return hostname; + } else if (!ips.isEmpty()) { + // If multiple IPs, show all + if (ips.size() > 1) { + return "[" + String.join(", ", ips) + "]"; + } else { + return ips.get(0); + } + } else { + return ip; + } } + /** + * @deprecated Use {@link #getIps()} instead to get all IP addresses. This method returns only + * the first IP for backward compatibility. + */ + @Deprecated public String getIp() { return this.ip; } + public List getIps() { + return new ArrayList<>(this.ips); + } + public String getHostname() { return this.hostname; } @@ -122,8 +158,25 @@ public int getTrancoRank() { return this.trancoRank; } + /** + * @deprecated Use {@link #setIps(List)} instead to set all IP addresses. This method is kept + * for backward compatibility. + */ + @Deprecated public void setIp(String ip) { this.ip = ip; + if (this.ips.isEmpty() || !this.ips.get(0).equals(ip)) { + this.ips = new ArrayList<>(); + this.ips.add(ip); + } + } + + public void setIps(List ips) { + this.ips = new ArrayList<>(ips); + // Set the first IP for backward compatibility + if (!ips.isEmpty()) { + this.ip = ips.get(0); + } } public void setHostname(String hostname) { diff --git a/src/main/java/de/rub/nds/crawler/denylist/DenylistFileProvider.java b/src/main/java/de/rub/nds/crawler/denylist/DenylistFileProvider.java index b480d2f..3999a28 100644 --- a/src/main/java/de/rub/nds/crawler/denylist/DenylistFileProvider.java +++ b/src/main/java/de/rub/nds/crawler/denylist/DenylistFileProvider.java @@ -69,9 +69,19 @@ private boolean isInSubnet(String ip, SubnetUtils.SubnetInfo subnetInfo) { @Override public synchronized boolean isDenylisted(ScanTarget target) { - return domainDenylistSet.contains(target.getHostname()) - || ipDenylistSet.contains(target.getIp()) - || cidrDenylist.stream() - .anyMatch(subnetInfo -> isInSubnet(target.getIp(), subnetInfo)); + // Check if hostname is denylisted + if (domainDenylistSet.contains(target.getHostname())) { + return true; + } + + // Check if any of the IPs are denylisted + for (String ip : target.getIps()) { + if (ipDenylistSet.contains(ip) + || cidrDenylist.stream().anyMatch(subnetInfo -> isInSubnet(ip, subnetInfo))) { + return true; + } + } + + return false; } } diff --git a/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java b/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java index 0cb002f..7fff9e6 100644 --- a/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java +++ b/src/main/java/de/rub/nds/crawler/persistence/MongoPersistenceProvider.java @@ -192,7 +192,9 @@ private JacksonMongoCollection initResultCollection( ScanResult.class, UuidRepresentation.STANDARD); // createIndex is idempotent, hence we do not need to check if an index exists - collection.createIndex(Indexes.ascending("scanTarget.ip")); + collection.createIndex( + Indexes.ascending("scanTarget.ip")); // Keep for backward compatibility + collection.createIndex(Indexes.ascending("scanTarget.ips")); // Index for multiple IPs collection.createIndex(Indexes.ascending("scanTarget.hostname")); collection.createIndex(Indexes.ascending("scanTarget.trancoRank")); collection.createIndex(Indexes.ascending("scanTarget.resultStatus")); diff --git a/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java b/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java new file mode 100644 index 0000000..6051512 --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java @@ -0,0 +1,134 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2022 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.data; + +import static org.junit.jupiter.api.Assertions.*; + +import de.rub.nds.crawler.constant.JobStatus; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; + +class ScanTargetTest { + + @Test + void testFromTargetStringWithHostname() { + // Test hostname resolution to multiple IPs + Pair result = ScanTarget.fromTargetString("localhost", 443, null); + + assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight()); + ScanTarget target = result.getLeft(); + + assertEquals("localhost", target.getHostname()); + assertEquals(443, target.getPort()); + assertNotNull(target.getIps()); + assertFalse(target.getIps().isEmpty()); + // localhost should resolve to at least one IP + assertTrue(target.getIps().size() >= 1); + // The deprecated getIp() should return the first IP + assertEquals(target.getIps().get(0), target.getIp()); + } + + @Test + void testFromTargetStringWithIpAddress() { + Pair result = ScanTarget.fromTargetString("127.0.0.1", 443, null); + + assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight()); + ScanTarget target = result.getLeft(); + + assertNull(target.getHostname()); + assertEquals("127.0.0.1", target.getIp()); + assertEquals(Arrays.asList("127.0.0.1"), target.getIps()); + assertEquals(443, target.getPort()); + } + + @Test + void testFromTargetStringWithPort() { + Pair result = + ScanTarget.fromTargetString("127.0.0.1:8443", 443, null); + + assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight()); + ScanTarget target = result.getLeft(); + + assertEquals("127.0.0.1", target.getIp()); + assertEquals(Arrays.asList("127.0.0.1"), target.getIps()); + assertEquals(8443, target.getPort()); + } + + @Test + void testFromTargetStringWithTrancoRank() { + Pair result = + ScanTarget.fromTargetString("100,127.0.0.1", 443, null); + + assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight()); + ScanTarget target = result.getLeft(); + + assertEquals(100, target.getTrancoRank()); + assertEquals("127.0.0.1", target.getIp()); + assertEquals(Arrays.asList("127.0.0.1"), target.getIps()); + } + + @Test + void testFromTargetStringUnresolvableHost() { + Pair result = + ScanTarget.fromTargetString("this-host-should-not-exist-12345.invalid", 443, null); + + assertEquals(JobStatus.UNRESOLVABLE, result.getRight()); + } + + @Test + void testSetIpsAndBackwardCompatibility() { + ScanTarget target = new ScanTarget(); + List ips = Arrays.asList("192.168.1.1", "192.168.1.2", "192.168.1.3"); + + target.setIps(ips); + + // Check that all IPs are stored + assertEquals(ips, target.getIps()); + // Check backward compatibility - getIp() should return the first IP + assertEquals("192.168.1.1", target.getIp()); + } + + @Test + void testSetIpBackwardCompatibility() { + ScanTarget target = new ScanTarget(); + + // Test deprecated setIp method + target.setIp("10.0.0.1"); + + assertEquals("10.0.0.1", target.getIp()); + assertEquals(Arrays.asList("10.0.0.1"), target.getIps()); + + // Setting another IP should update the list + target.setIp("10.0.0.2"); + assertEquals("10.0.0.2", target.getIp()); + assertEquals(Arrays.asList("10.0.0.2"), target.getIps()); + } + + @Test + void testToStringWithMultipleIps() { + ScanTarget target = new ScanTarget(); + + // Test with hostname + target.setHostname("example.com"); + target.setIps(Arrays.asList("192.168.1.1", "192.168.1.2")); + assertEquals("example.com", target.toString()); + + // Test with single IP (no hostname) + target = new ScanTarget(); + target.setIps(Arrays.asList("192.168.1.1")); + assertEquals("192.168.1.1", target.toString()); + + // Test with multiple IPs (no hostname) + target = new ScanTarget(); + target.setIps(Arrays.asList("192.168.1.1", "192.168.1.2", "192.168.1.3")); + assertEquals("[192.168.1.1, 192.168.1.2, 192.168.1.3]", target.toString()); + } +} diff --git a/src/test/java/de/rub/nds/crawler/denylist/DenylistFileProviderTest.java b/src/test/java/de/rub/nds/crawler/denylist/DenylistFileProviderTest.java new file mode 100644 index 0000000..44869de --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/denylist/DenylistFileProviderTest.java @@ -0,0 +1,142 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2022 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.denylist; + +import static org.junit.jupiter.api.Assertions.*; + +import de.rub.nds.crawler.data.ScanTarget; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Arrays; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DenylistFileProviderTest { + + @TempDir File tempDir; + + private File denylistFile; + private DenylistFileProvider provider; + + @BeforeEach + void setUp() throws IOException { + denylistFile = new File(tempDir, "denylist.txt"); + writeDenylistFile(); + provider = new DenylistFileProvider(denylistFile.getAbsolutePath()); + } + + @AfterEach + void tearDown() { + if (denylistFile.exists()) { + denylistFile.delete(); + } + } + + private void writeDenylistFile() throws IOException { + try (FileWriter writer = new FileWriter(denylistFile)) { + // Domain denylists + writer.write("example.com\n"); + writer.write("blocked-domain.org\n"); + // IP denylists + writer.write("192.168.1.100\n"); + writer.write("10.0.0.50\n"); + // CIDR denylists + writer.write("172.16.0.0/16\n"); + writer.write("203.0.113.0/24\n"); + } + } + + @Test + void testIsDenylistedWithBlockedDomain() { + ScanTarget target = new ScanTarget(); + target.setHostname("example.com"); + target.setIps(Arrays.asList("1.2.3.4", "5.6.7.8")); + + assertTrue(provider.isDenylisted(target)); + } + + @Test + void testIsDenylistedWithAllowedDomain() { + ScanTarget target = new ScanTarget(); + target.setHostname("allowed-domain.com"); + target.setIps(Arrays.asList("1.2.3.4", "5.6.7.8")); + + assertFalse(provider.isDenylisted(target)); + } + + @Test + void testIsDenylistedWithBlockedSingleIp() { + ScanTarget target = new ScanTarget(); + target.setHostname("allowed-domain.com"); + target.setIps(Arrays.asList("192.168.1.100")); + + assertTrue(provider.isDenylisted(target)); + } + + @Test + void testIsDenylistedWithMultipleIpsOneBlocked() { + ScanTarget target = new ScanTarget(); + target.setHostname("allowed-domain.com"); + // One IP is blocked, others are not + target.setIps(Arrays.asList("1.2.3.4", "192.168.1.100", "5.6.7.8")); + + assertTrue(provider.isDenylisted(target)); + } + + @Test + void testIsDenylistedWithMultipleIpsNoneBlocked() { + ScanTarget target = new ScanTarget(); + target.setHostname("allowed-domain.com"); + target.setIps(Arrays.asList("1.2.3.4", "5.6.7.8", "9.10.11.12")); + + assertFalse(provider.isDenylisted(target)); + } + + @Test + void testIsDenylistedWithCidrBlockedIp() { + ScanTarget target = new ScanTarget(); + target.setHostname("allowed-domain.com"); + // IP is in the 172.16.0.0/16 CIDR range + target.setIps(Arrays.asList("172.16.5.10")); + + assertTrue(provider.isDenylisted(target)); + } + + @Test + void testIsDenylistedWithMultipleIpsOneCidrBlocked() { + ScanTarget target = new ScanTarget(); + target.setHostname("allowed-domain.com"); + // One IP is in CIDR range, others are not + target.setIps(Arrays.asList("1.2.3.4", "203.0.113.50", "5.6.7.8")); + + assertTrue(provider.isDenylisted(target)); + } + + @Test + void testIsDenylistedWithEmptyIpsList() { + ScanTarget target = new ScanTarget(); + target.setHostname("allowed-domain.com"); + target.setIps(Arrays.asList()); + + assertFalse(provider.isDenylisted(target)); + } + + @Test + void testBackwardCompatibilityWithDeprecatedIpField() { + ScanTarget target = new ScanTarget(); + target.setHostname("allowed-domain.com"); + // Use deprecated setIp method + target.setIp("10.0.0.50"); + + assertTrue(provider.isDenylisted(target)); + } +}