Skip to content

Commit a212421

Browse files
jakejarvisjakejarvis
committed
feat: implement RDAP bootstrap data fetching with caching for improved domain lookup efficiency
1 parent 55e63d1 commit a212421

File tree

12 files changed

+164
-127
lines changed

12 files changed

+164
-127
lines changed

‎AGENTS.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@
4848
- Global setup in `vitest.setup.ts`:
4949
- Mocks analytics clients/servers (`@/lib/analytics/server` and `@/lib/analytics/client`).
5050
- Mocks `server-only` module.
51-
- `unstable_cache` mocked as a no-op; caching behavior is not under test.
5251
- Database in tests: Drizzle client is not globally mocked. Replace `@/server/db/client` with a PGlite-backed instance when needed (`@/lib/db/pglite`).
5352
- Redis in tests: do NOT use globals. Mock per-suite with the in-memory adapter:
5453
- In `beforeAll`: `const { makeInMemoryRedis } = await import("@/lib/redis-mock"); const impl = makeInMemoryRedis(); vi.doMock("@/lib/redis", () => impl);`

‎lib/cloudflare.test.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
/* @vitest-environment node */
22
import { afterEach, describe, expect, it, vi } from "vitest";
3-
import { isCloudflareIpAsync } from "./cloudflare";
3+
import { isCloudflareIp } from "./cloudflare";
44

5-
describe("isCloudflareIpAsync", () => {
5+
describe("isCloudflareIp", () => {
66
afterEach(() => {
77
vi.restoreAllMocks();
88
});
@@ -19,10 +19,10 @@ describe("isCloudflareIpAsync", () => {
1919
}),
2020
);
2121

22-
expect(await isCloudflareIpAsync("1.2.3.4")).toBe(true);
23-
expect(await isCloudflareIpAsync("5.6.7.8")).toBe(false);
24-
expect(await isCloudflareIpAsync("2001:db8::1")).toBe(true);
25-
expect(await isCloudflareIpAsync("2001:dead::1")).toBe(false);
22+
expect(await isCloudflareIp("1.2.3.4")).toBe(true);
23+
expect(await isCloudflareIp("5.6.7.8")).toBe(false);
24+
expect(await isCloudflareIp("2001:db8::1")).toBe(true);
25+
expect(await isCloudflareIp("2001:dead::1")).toBe(false);
2626
fetchMock.mockRestore();
2727
});
2828
});

‎lib/cloudflare.ts‎

Lines changed: 65 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import * as ipaddr from "ipaddr.js";
2-
import { unstable_cache } from "next/cache";
2+
import { CLOUDFLARE_IPS_URL, TTL_CLOUDFLARE_IPS } from "@/lib/constants";
33

44
export interface CloudflareIpRanges {
55
ipv4Cidrs: string[];
@@ -9,76 +9,80 @@ export interface CloudflareIpRanges {
99
let lastLoadedIpv4Parsed: Array<[ipaddr.IPv4, number]> | undefined;
1010
let lastLoadedIpv6Parsed: Array<[ipaddr.IPv6, number]> | undefined;
1111

12-
export const getCloudflareIpRanges = unstable_cache(
13-
async (): Promise<CloudflareIpRanges> => {
14-
const res = await fetch("https://api.cloudflare.com/client/v4/ips");
15-
if (!res.ok) {
16-
throw new Error(`Failed to fetch Cloudflare IPs: ${res.status}`);
17-
}
18-
const data = await res.json();
12+
async function getCloudflareIpRanges(): Promise<CloudflareIpRanges> {
13+
const res = await fetch(CLOUDFLARE_IPS_URL, {
14+
next: { revalidate: TTL_CLOUDFLARE_IPS },
15+
});
1916

20-
const ranges: CloudflareIpRanges = {
21-
ipv4Cidrs: data.result?.ipv4_cidrs || [],
22-
ipv6Cidrs: data.result?.ipv6_cidrs || [],
23-
};
17+
if (!res.ok) {
18+
throw new Error(`Failed to fetch Cloudflare IPs: ${res.status}`);
19+
}
2420

25-
// Pre-parse IPv4 CIDRs for fast sync/async checks
26-
try {
27-
lastLoadedIpv4Parsed = ranges.ipv4Cidrs
28-
.map((cidr) => {
29-
try {
30-
const [net, prefix] = ipaddr.parseCIDR(cidr);
31-
if (net.kind() !== "ipv4") return undefined;
32-
return [net as ipaddr.IPv4, prefix] as [ipaddr.IPv4, number];
33-
} catch {
34-
return undefined;
35-
}
36-
})
37-
.filter(Boolean) as Array<[ipaddr.IPv4, number]>;
38-
} catch {
39-
lastLoadedIpv4Parsed = undefined;
40-
}
21+
const data = await res.json();
4122

42-
// Pre-parse IPv6 CIDRs for fast sync/async checks
43-
try {
44-
lastLoadedIpv6Parsed = ranges.ipv6Cidrs
45-
.map((cidr) => {
46-
try {
47-
const [net, prefix] = ipaddr.parseCIDR(cidr);
48-
if (net.kind() !== "ipv6") return undefined;
49-
return [net as ipaddr.IPv6, prefix] as [ipaddr.IPv6, number];
50-
} catch {
51-
return undefined;
52-
}
53-
})
54-
.filter(Boolean) as Array<[ipaddr.IPv6, number]>;
55-
} catch {
56-
lastLoadedIpv6Parsed = undefined;
57-
}
58-
return ranges;
59-
},
60-
["cloudflare-ip-ranges"],
61-
// Cache for a very long time (30 days)
62-
{ revalidate: 30 * 24 * 60 * 60 },
63-
);
23+
const ranges: CloudflareIpRanges = {
24+
ipv4Cidrs: data.result?.ipv4_cidrs || [],
25+
ipv6Cidrs: data.result?.ipv6_cidrs || [],
26+
};
6427

65-
export function isCloudflareIp(ip: string): boolean {
66-
if (ipaddr.IPv4.isValid(ip)) {
67-
if (!lastLoadedIpv4Parsed) return false;
68-
const v4 = ipaddr.IPv4.parse(ip);
69-
return lastLoadedIpv4Parsed.some((range) => v4.match(range));
28+
// Pre-parse IPv4 CIDRs for fast sync/async checks
29+
try {
30+
lastLoadedIpv4Parsed = ranges.ipv4Cidrs
31+
.map((cidr) => {
32+
try {
33+
const [net, prefix] = ipaddr.parseCIDR(cidr);
34+
if (net.kind() !== "ipv4") return undefined;
35+
return [net as ipaddr.IPv4, prefix] as [ipaddr.IPv4, number];
36+
} catch {
37+
return undefined;
38+
}
39+
})
40+
.filter(Boolean) as Array<[ipaddr.IPv4, number]>;
41+
} catch {
42+
lastLoadedIpv4Parsed = undefined;
7043
}
7144

72-
if (ipaddr.IPv6.isValid(ip)) {
73-
if (!lastLoadedIpv6Parsed) return false;
74-
const v6 = ipaddr.IPv6.parse(ip);
75-
return lastLoadedIpv6Parsed.some((range) => v6.match(range));
45+
// Pre-parse IPv6 CIDRs for fast sync/async checks
46+
try {
47+
lastLoadedIpv6Parsed = ranges.ipv6Cidrs
48+
.map((cidr) => {
49+
try {
50+
const [net, prefix] = ipaddr.parseCIDR(cidr);
51+
if (net.kind() !== "ipv6") return undefined;
52+
return [net as ipaddr.IPv6, prefix] as [ipaddr.IPv6, number];
53+
} catch {
54+
return undefined;
55+
}
56+
})
57+
.filter(Boolean) as Array<[ipaddr.IPv6, number]>;
58+
} catch {
59+
lastLoadedIpv6Parsed = undefined;
7660
}
7761

78-
return false;
62+
return ranges;
7963
}
8064

81-
export async function isCloudflareIpAsync(ip: string): Promise<boolean> {
65+
function ipV4InCidr(addr: ipaddr.IPv4, cidr: string): boolean {
66+
try {
67+
const [net, prefix] = ipaddr.parseCIDR(cidr);
68+
if (net.kind() !== "ipv4") return false;
69+
return addr.match([net as ipaddr.IPv4, prefix]);
70+
} catch {
71+
return false;
72+
}
73+
}
74+
75+
function ipV6InCidr(addr: ipaddr.IPv6, cidr: string): boolean {
76+
try {
77+
const [net, prefix] = ipaddr.parseCIDR(cidr);
78+
if (net.kind() !== "ipv6") return false;
79+
return addr.match([net as ipaddr.IPv6, prefix]);
80+
} catch {
81+
return false;
82+
}
83+
}
84+
85+
export async function isCloudflareIp(ip: string): Promise<boolean> {
8286
const ranges = await getCloudflareIpRanges();
8387

8488
if (ipaddr.IPv4.isValid(ip)) {
@@ -101,23 +105,3 @@ export async function isCloudflareIpAsync(ip: string): Promise<boolean> {
101105

102106
return false;
103107
}
104-
105-
function ipV4InCidr(addr: ipaddr.IPv4, cidr: string): boolean {
106-
try {
107-
const [net, prefix] = ipaddr.parseCIDR(cidr);
108-
if (net.kind() !== "ipv4") return false;
109-
return addr.match([net as ipaddr.IPv4, prefix]);
110-
} catch {
111-
return false;
112-
}
113-
}
114-
115-
function ipV6InCidr(addr: ipaddr.IPv6, cidr: string): boolean {
116-
try {
117-
const [net, prefix] = ipaddr.parseCIDR(cidr);
118-
if (net.kind() !== "ipv6") return false;
119-
return addr.match([net as ipaddr.IPv6, prefix]);
120-
} catch {
121-
return false;
122-
}
123-
}

‎lib/constants.ts‎

Lines changed: 42 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,44 +8,67 @@ export const USER_AGENT =
88

99
export const REPOSITORY_SLUG = "jakejarvis/domainstack.io";
1010

11-
// Time constants
12-
const SECONDS_PER_HOUR = 60 * 60;
13-
const SECONDS_PER_DAY = 24 * SECONDS_PER_HOUR;
11+
/**
12+
* RDAP Bootstrap Registry URL from IANA.
13+
* This JSON file maps TLDs to their authoritative RDAP servers.
14+
* @see https://datatracker.ietf.org/doc/html/rfc7484
15+
*/
16+
export const RDAP_BOOTSTRAP_URL = "https://data.iana.org/rdap/dns.json";
17+
18+
/**
19+
* Cloudflare IP Ranges URL.
20+
* This JSON file contains the IP ranges for Cloudflare's network.
21+
* @see https://developers.cloudflare.com/api/resources/ips/methods/list/
22+
*/
23+
export const CLOUDFLARE_IPS_URL = "https://api.cloudflare.com/client/v4/ips";
24+
25+
// Time constants (in seconds)
26+
const ONE_HOUR = 60 * 60;
27+
const ONE_DAY = 24 * ONE_HOUR;
28+
const ONE_WEEK = 7 * ONE_DAY;
29+
30+
// RDAP bootstrap data (dns.json from IANA)
31+
// Changes infrequently (new TLDs, server updates); safe to cache aggressively
32+
export const TTL_RDAP_BOOTSTRAP = ONE_DAY; // 24 hours
33+
34+
// Cloudflare IP ranges
35+
// Changes infrequently (new IP ranges); safe to cache aggressively
36+
export const TTL_CLOUDFLARE_IPS = ONE_DAY; // 24 hours
1437

1538
// ===== Blob Storage Cache TTLs =====
1639
// How long to cache uploaded assets (favicons, screenshots, social images)
17-
export const TTL_FAVICON = 7 * SECONDS_PER_DAY; // 1 week
18-
export const TTL_SCREENSHOT = 14 * SECONDS_PER_DAY; // 2 weeks
19-
export const TTL_SOCIAL_PREVIEW = 7 * SECONDS_PER_DAY; // 1 week
40+
export const TTL_FAVICON = ONE_WEEK; // 1 week
41+
export const TTL_SCREENSHOT = 2 * ONE_WEEK; // 2 weeks
42+
export const TTL_SOCIAL_PREVIEW = ONE_WEEK; // 1 week
2043

2144
// ===== Database Cache Expiry TTLs =====
2245
// When cached data in Postgres becomes stale and needs refresh.
2346
// Used by lib/db/ttl.ts functions to calculate expiresAt timestamps.
2447

2548
// Registration data
26-
export const TTL_REGISTRATION_REGISTERED = SECONDS_PER_DAY; // 24 hours
27-
export const TTL_REGISTRATION_NEAR_EXPIRY = SECONDS_PER_HOUR; // 1 hour (aggressive near expiry)
28-
export const TTL_REGISTRATION_EXPIRY_THRESHOLD = 7 * SECONDS_PER_DAY; // 7 days (when to switch to aggressive)
49+
export const TTL_REGISTRATION_REGISTERED = ONE_DAY; // 24 hours
50+
export const TTL_REGISTRATION_NEAR_EXPIRY = ONE_HOUR; // 1 hour (aggressive near expiry)
51+
export const TTL_REGISTRATION_EXPIRY_THRESHOLD = ONE_WEEK; // 7 days (when to switch to aggressive)
2952

3053
// DNS records
31-
export const TTL_DNS_DEFAULT = SECONDS_PER_HOUR; // 1 hour (fallback when no TTL provided)
32-
export const TTL_DNS_MAX = SECONDS_PER_DAY; // 24 hours (cap for received TTLs)
54+
export const TTL_DNS_DEFAULT = ONE_HOUR; // 1 hour (fallback when no TTL provided)
55+
export const TTL_DNS_MAX = ONE_DAY; // 24 hours (cap for received TTLs)
3356

3457
// TLS certificates
35-
export const TTL_CERTIFICATES_WINDOW = SECONDS_PER_DAY; // 24 hours (normal refresh window)
36-
export const TTL_CERTIFICATES_MIN = SECONDS_PER_HOUR; // 1 hour (minimum check interval)
37-
export const TTL_CERTIFICATES_EXPIRY_BUFFER = 48 * SECONDS_PER_HOUR; // 48 hours (start aggressive checking before expiry)
58+
export const TTL_CERTIFICATES_WINDOW = ONE_DAY; // 24 hours (normal refresh window)
59+
export const TTL_CERTIFICATES_MIN = ONE_HOUR; // 1 hour (minimum check interval)
60+
export const TTL_CERTIFICATES_EXPIRY_BUFFER = 2 * ONE_DAY; // 48 hours (start aggressive checking before expiry)
3861

3962
// HTTP headers, hosting, SEO
40-
export const TTL_HEADERS = 12 * SECONDS_PER_HOUR; // 12 hours
41-
export const TTL_HOSTING = SECONDS_PER_DAY; // 24 hours
42-
export const TTL_SEO = SECONDS_PER_DAY; // 24 hours
63+
export const TTL_HEADERS = 12 * ONE_HOUR; // 12 hours
64+
export const TTL_HOSTING = ONE_DAY; // 24 hours
65+
export const TTL_SEO = ONE_DAY; // 24 hours
4366

4467
// ===== Redis Cache TTLs =====
4568
// Lightweight registration status cache (stores only true/false, not full data).
4669
// Unregistered domains are ONLY cached here, never in Postgres.
47-
export const REDIS_TTL_REGISTERED = SECONDS_PER_DAY; // 24 hours
48-
export const REDIS_TTL_UNREGISTERED = SECONDS_PER_HOUR; // 1 hour
70+
export const REDIS_TTL_REGISTERED = ONE_DAY; // 24 hours
71+
export const REDIS_TTL_UNREGISTERED = ONE_HOUR; // 1 hour
4972

5073
// ===== Background Job Revalidation =====
5174
// How often Inngest jobs attempt to refresh each section's data.

‎lib/rdap-bootstrap.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import "server-only";
2+
import type { BootstrapData } from "rdapper";
3+
import { RDAP_BOOTSTRAP_URL, TTL_RDAP_BOOTSTRAP } from "@/lib/constants";
4+
5+
/**
6+
* Fetch RDAP bootstrap data with Next.js caching.
7+
*
8+
* The bootstrap registry changes infrequently (new TLDs, server updates),
9+
* so we cache it for 24 hours using Next.js's built-in fetch cache.
10+
*
11+
* This eliminates redundant fetches to IANA on every domain lookup when
12+
* passed to rdapper's lookup() via the customBootstrapData option.
13+
*
14+
* @returns RDAP bootstrap data containing TLD-to-server mappings
15+
* @throws Error if fetch fails (caller should handle or let rdapper fetch directly)
16+
*/
17+
export async function getRdapBootstrapData(): Promise<BootstrapData> {
18+
const res = await fetch(RDAP_BOOTSTRAP_URL, {
19+
next: { revalidate: TTL_RDAP_BOOTSTRAP },
20+
});
21+
22+
if (!res.ok) {
23+
throw new Error(
24+
`Failed to fetch RDAP bootstrap: ${res.status} ${res.statusText}`,
25+
);
26+
}
27+
28+
return res.json();
29+
}

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@
7474
"posthog-node": "^5.10.4",
7575
"puppeteer-core": "24.26.1",
7676
"radix-ui": "^1.4.3",
77-
"rdapper": "^0.10.4",
77+
"rdapper": "^0.11.0",
7878
"react": "19.2.0",
7979
"react-dom": "19.2.0",
8080
"react-map-gl": "^8.1.0",

‎pnpm-lock.yaml‎

Lines changed: 5 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎server/services/dns.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
} from "vitest";
1111

1212
vi.mock("@/lib/cloudflare", () => ({
13-
isCloudflareIpAsync: vi.fn(async () => false),
13+
isCloudflareIp: vi.fn(async () => false),
1414
}));
1515

1616
beforeAll(async () => {

‎server/services/dns.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { eq } from "drizzle-orm";
22
import { acquireLockOrWaitForResult } from "@/lib/cache";
3-
import { isCloudflareIpAsync } from "@/lib/cloudflare";
3+
import { isCloudflareIp } from "@/lib/cloudflare";
44
import { USER_AGENT } from "@/lib/constants";
55
import { db } from "@/lib/db/client";
66
import { replaceDns } from "@/lib/db/repos/dns";
@@ -485,7 +485,7 @@ function normalizeAnswer(
485485
case "A":
486486
case "AAAA": {
487487
const value = trimDot(a.data);
488-
const isCloudflarePromise = isCloudflareIpAsync(value);
488+
const isCloudflarePromise = isCloudflareIp(value);
489489
return isCloudflarePromise.then((isCloudflare) => ({
490490
type,
491491
name,

0 commit comments

Comments
 (0)