Add unknown protocol tunnelling & raw-passthrough-opened/closed events

Author: pimterry

src/admin/mockttp-admin-model.ts

@@ -32,6 +32,8 @@ const TLS_PASSTHROUGH_OPENED_TOPIC = 'tls-passthrough-opened';
 const TLS_PASSTHROUGH_CLOSED_TOPIC = 'tls-passthrough-closed';
 const TLS_CLIENT_ERROR_TOPIC = 'tls-client-error';
 const CLIENT_ERROR_TOPIC = 'client-error';
+const RAW_PASSTHROUGH_OPENED_TOPIC = 'raw-passthrough-opened';
+const RAW_PASSTHROUGH_CLOSED_TOPIC = 'raw-passthrough-closed';
 const RULE_EVENT_TOPIC = 'rule-event';
 
 async function buildMockedEndpointData(endpoint: ServerMockedEndpoint): Promise<MockedEndpointData> {
@@ -132,6 +134,18 @@ export function buildAdminServerModel(
         })
     });
 
+    mockServer.on('raw-passthrough-opened', (evt) => {
+        pubsub.publish(RAW_PASSTHROUGH_OPENED_TOPIC, {
+            rawPassthroughOpened: evt
+        })
+    });
+
+    mockServer.on('raw-passthrough-closed', (evt) => {
+        pubsub.publish(RAW_PASSTHROUGH_CLOSED_TOPIC, {
+            rawPassthroughClosed: evt
+        })
+    });
+
     mockServer.on('rule-event', (evt) => {
         pubsub.publish(RULE_EVENT_TOPIC, {
             ruleEvent: evt
@@ -237,6 +251,12 @@ export function buildAdminServerModel(
             failedClientRequest: {
                 subscribe: () => pubsub.asyncIterator(CLIENT_ERROR_TOPIC)
             },
+            rawPassthroughOpened: {
+                subscribe: () => pubsub.asyncIterator(RAW_PASSTHROUGH_OPENED_TOPIC)
+            },
+            rawPassthroughClosed: {
+                subscribe: () => pubsub.asyncIterator(RAW_PASSTHROUGH_CLOSED_TOPIC)
+            },
             ruleEvent: {
                 subscribe: () => pubsub.asyncIterator(RULE_EVENT_TOPIC)
             }

src/admin/mockttp-schema.ts

@@ -31,6 +31,8 @@ export const MockttpSchema = gql`
         tlsPassthroughOpened: TlsPassthroughEvent!
         tlsPassthroughClosed: TlsPassthroughEvent!
         failedTlsRequest: TlsHandshakeFailure!
+        rawPassthroughOpened: RawPassthroughEvent!
+        rawPassthroughClosed: RawPassthroughEvent!
         failedClientRequest: ClientError!
         ruleEvent: RuleEvent!
     }
@@ -60,6 +62,8 @@ export const MockttpSchema = gql`
 
     type TlsPassthroughEvent {
         id: String!
+
+        upstreamHost: String
         upstreamPort: Int!
 
         hostname: String
@@ -114,6 +118,18 @@ export const MockttpSchema = gql`
         remotePort: Int
     }
 
+    type RawPassthroughEvent {
+        id: String!
+
+        upstreamHost: String!
+        upstreamPort: Int!
+
+        remoteIpAddress: String!
+        remotePort: Int!
+        tags: [String!]!
+        timingEvents: Json!
+    }
+
     type RuleEvent {
         requestId: ID!
         ruleId: ID!

src/client/mockttp-admin-request-builder.ts

@@ -379,6 +379,8 @@ export class MockttpAdminRequestBuilder {
             'tls-passthrough-opened': gql`subscription OnTlsPassthroughOpened {
                 tlsPassthroughOpened {
                     id
+
+                    ${this.schema.asOptionalField('TlsPassthroughEvent', 'upstreamHost')}
                     upstreamPort
 
                     hostname
@@ -392,6 +394,8 @@ export class MockttpAdminRequestBuilder {
             'tls-passthrough-closed': gql`subscription OnTlsPassthroughClosed {
                 tlsPassthroughClosed {
                     id
+
+                    ${this.schema.asOptionalField('TlsPassthroughEvent', 'upstreamHost')}
                     upstreamPort
 
                     hostname
@@ -451,6 +455,32 @@ export class MockttpAdminRequestBuilder {
                     }
                 }
             }`,
+            'raw-passthrough-opened': gql`subscription OnRawPassthroughOpened {
+                rawPassthroughOpened {
+                    id
+
+                    upstreamHost
+                    upstreamPort
+
+                    remoteIpAddress
+                    remotePort
+                    tags
+                    timingEvents
+                }
+            }`,
+            'raw-passthrough-closed': gql`subscription OnRawPassthroughClosed {
+                rawPassthroughClosed {
+                    id
+
+                    upstreamHost
+                    upstreamPort
+
+                    remoteIpAddress
+                    remotePort
+                    tags
+                    timingEvents
+                }
+            }`,
             'rule-event': gql`subscription OnRuleEvent {
                 ruleEvent {
                     requestId

src/mockttp.ts

@@ -20,7 +20,8 @@ import {
     WebSocketMessage,
     WebSocketClose,
     AbortedRequest,
-    RuleEvent
+    RuleEvent,
+    RawPassthroughEvent
 } from "./types";
 import type { RequestRuleData } from "./rules/requests/request-rule";
 import type { WebSocketRuleData } from "./rules/websockets/websocket-rule";
@@ -545,6 +546,30 @@ export interface Mockttp {
      */
     on(event: 'client-error', callback: (error: ClientError) => void): Promise<void>;
 
+    /**
+     * Subscribe to hear about connections that are passed through the proxy without
+     * interception, due to the `passthrough` option.
+     *
+     * This is separate to TLS passthrough: raw passthrough happens automatically
+     * before any TLS handshake is received (so includes no TLS data, and may use any
+     * protocol) generally because the protocol on the connection is not HTTP. TLS
+     * passthrough happens after the TLS client hello has been received, only if it
+     * has matched a rule defined in the tlsPassthrough options (e.g. a specific
+     * hostname).
+     *
+     * @category Events
+     */
+    on(event: 'raw-passthrough-opened', callback: (req: RawPassthroughEvent) => void): Promise<void>;
+
+    /**
+     * Subscribe to hear about close of connections that are passed through the proxy
+     * without interception, due to the `passthrough` option. See `raw-passthrough-opened`
+     * for more details.
+     *
+     * @category Events
+     */
+    on(event: 'raw-passthrough-closed', callback: (req: RawPassthroughEvent) => void): Promise<void>;
+
     /**
      * Some rules may emit events with metadata about request processing. For example,
      * passthrough rules may emit events about upstream server interactions.
@@ -783,6 +808,29 @@ export interface MockttpOptions {
      */
     socks?: boolean;
 
+    /**
+     * An array of rules for traffic that should be passed through the proxy
+     * immediately, without interception or modification.
+     *
+     * This is subtly different to TLS passthrough/interceptOnly, which only
+     * apply to TLS connections, and only after the TLS client hello has been
+     * received and found to match a rule.
+     *
+     * For now, the only rule here is 'unknown-protocol', which enables
+     * passthrough of all unknown protocols (i.e. traffic that is definitely
+     * not HTTP, HTTP/2, WebSocket, or SOCKS traffic) which are received on
+     * a proxy connection (a connection carrying end-destination information,
+     * such as SOCKS - direct connections of unknown data without any final
+     * destination information from a preceeding tunnel cannot be passed
+     * through).
+     *
+     * Unknown protocol connections that cannot be passed through (because
+     * this rule is not enabled, or because they are not proxied with a
+     * destination specified) will be closed with a 400 Bad Request HTTP
+     * response like any other client HTTP error.
+     */
+    passthrough?: Array<'unknown-protocol'>;
+
     /**
      * By default, requests that match no rules will receive an explanation of the
      * request & existing rules, followed by some suggested example Mockttp code
@@ -834,6 +882,8 @@ export type SubscribableEvent =
     | 'tls-passthrough-closed'
     | 'tls-client-error'
     | 'client-error'
+    | 'raw-passthrough-opened'
+    | 'raw-passthrough-closed'
     | 'rule-event';
 
 /**

src/server/http-combo-server.ts

@@ -4,7 +4,6 @@ import net = require('net');
 import tls = require('tls');
 import http = require('http');
 import http2 = require('http2');
-import * as streams from 'stream';
 
 import * as semver from 'semver';
 import { makeDestroyable, DestroyableServer } from 'destroyable-server';
@@ -24,15 +23,17 @@ import { shouldPassThrough } from '../util/server-utils';
 import {
     getParentSocket,
     buildSocketTimingInfo,
-    buildSocketEventData,
+    buildTlsSocketEventData,
     SocketIsh,
     InitialRemoteAddress,
     InitialRemotePort,
     SocketTimingInfo,
     LastTunnelAddress,
     LastHopEncrypted,
     TlsMetadata,
-    TlsSetupCompleted
+    TlsSetupCompleted,
+    getAddressAndPort,
+    resetOrDestroy
 } from '../util/socket-util';
 import { MockttpHttpsOptions } from '../mockttp';
 import { buildSocksServer, SocksTcpAddress } from './socks-server';
@@ -62,13 +63,6 @@ const originalSocketInit = (<any>tls.TLSSocket.prototype)._init;
     };
 };
 
-export interface ComboServerOptions {
-    debug: boolean;
-    https: MockttpHttpsOptions | undefined;
-    http2: boolean | 'fallback';
-    socks: boolean;
-};
-
 // Takes an established TLS socket, calls the error listener if it's silently closed
 function ifTlsDropped(socket: tls.TLSSocket, errorCallback: () => void) {
     new Promise((resolve, reject) => {
@@ -139,26 +133,35 @@ function buildTlsError(
     socket: tls.TLSSocket,
     cause: TlsHandshakeFailure['failureCause']
 ): TlsHandshakeFailure {
-    const eventData = buildSocketEventData(socket) as TlsHandshakeFailure;
+    const eventData = buildTlsSocketEventData(socket) as TlsHandshakeFailure;
 
     eventData.failureCause = cause;
     eventData.timingEvents.failureTimestamp = now();
 
     return eventData;
 }
 
+export interface ComboServerOptions {
+    debug: boolean;
+    https: MockttpHttpsOptions | undefined;
+    http2: boolean | 'fallback';
+    socks: boolean;
+    passthroughUnknownProtocols: boolean;
+
+    requestListener: (req: http.IncomingMessage, res: http.ServerResponse) => void;
+    tlsClientErrorListener: (socket: tls.TLSSocket, req: TlsHandshakeFailure) => void;
+    tlsPassthroughListener: (socket: net.Socket, address: string, port?: number) => void;
+    rawPassthroughListener: (socket: net.Socket, address: string, port?: number) => void;
+};
+
 // The low-level server that handles all the sockets & TLS. The server will correctly call the
 // given handler for both HTTP & HTTPS direct connections, or connections when used as an
 // either HTTP or HTTPS proxy, all on the same port.
-export async function createComboServer(
-    options: ComboServerOptions,
-    requestListener: (req: http.IncomingMessage, res: http.ServerResponse) => void,
-    tlsClientErrorListener: (socket: tls.TLSSocket, req: TlsHandshakeFailure) => void,
-    tlsPassthroughListener: (socket: net.Socket, address: string, port?: number) => void
-): Promise<DestroyableServer<net.Server>> {
+export async function createComboServer(options: ComboServerOptions): Promise<DestroyableServer<net.Server>> {
     let server: net.Server;
     let tlsServer: tls.Server | undefined = undefined;
     let socksServer: net.Server | undefined = undefined;
+    let unknownProtocolServer: net.Server | undefined = undefined;
 
     if (options.https) {
         const ca = await getCA(options.https);
@@ -217,7 +220,7 @@ export async function createComboServer(
             tlsServer,
             options.https.tlsPassthrough,
             options.https.tlsInterceptOnly,
-            tlsPassthroughListener
+            options.tlsPassthroughListener
         );
     }
 
@@ -243,10 +246,29 @@ export async function createComboServer(
         });
     }
 
+    if (options.passthroughUnknownProtocols) {
+        unknownProtocolServer = net.createServer((socket) => {
+            const destination = socket[LastTunnelAddress];
+            if (!destination) {
+                server.emit('clientError', new Error('Unknown protocol without destination'), socket);
+                return;
+            }
+
+            const [host, port] = getAddressAndPort(destination);
+            if (!port) { // Both CONNECT & SOCKS require a port, so this shouldn't happen
+                server.emit('clientError', new Error('Unknown protocol without destination port'), socket);
+                return;
+            }
+
+            options.rawPassthroughListener(socket, host, port);
+        });
+    }
+
     server = httpolyglot.createServer({
         tls: tlsServer,
         socks: socksServer,
-    }, requestListener);
+        unknownProtocol: unknownProtocolServer
+    }, options.requestListener);
 
     // In Node v20, this option was added, rejecting all requests with no host header. While that's good, in
     // our case, we want to handle the garbage requests too, so we disable it:
@@ -282,7 +304,7 @@ export async function createComboServer(
 
         socket[LastHopEncrypted] = true;
         ifTlsDropped(socket, () => {
-            tlsClientErrorListener(socket, buildTlsError(socket, 'closed'));
+            options.tlsClientErrorListener(socket, buildTlsError(socket, 'closed'));
         });
     });
 
@@ -295,7 +317,7 @@ export async function createComboServer(
     });
 
     server.on('tlsClientError', (error: Error, socket: tls.TLSSocket) => {
-        tlsClientErrorListener(socket, buildTlsError(socket, getCauseFromError(error)));
+        options.tlsClientErrorListener(socket, buildTlsError(socket, getCauseFromError(error)));
     });
 
     // If the server receives a HTTP/HTTPS CONNECT request, Pretend to tunnel, then just re-handle:
@@ -431,30 +453,23 @@ function analyzeAndMaybePassThroughTls(
 
             // SNI is a good clue for where the request is headed, but an explicit proxy address (via
             // CONNECT or SOCKS) is even better. Note that this may be a hostname or IPv4/6 address:
-            let connectHostname: string | undefined;
-            let connectPort: string | undefined;
+            let upstreamHostname: string | undefined;
+            let upstreamPort: number | undefined;
             if (socket[LastTunnelAddress]) {
-                const lastColonIndex = socket[LastTunnelAddress].lastIndexOf(':');
-                if (lastColonIndex !== -1) {
-                    connectHostname = socket[LastTunnelAddress].slice(0, lastColonIndex);
-                    connectPort = socket[LastTunnelAddress].slice(lastColonIndex + 1);
-                } else {
-                    connectHostname = socket[LastTunnelAddress];
-                }
+                ([upstreamHostname, upstreamPort] = getAddressAndPort(socket[LastTunnelAddress]));
             }
 
             socket[TlsMetadata] = {
                 sniHostname,
-                connectHostname,
-                connectPort,
+                connectHostname: upstreamHostname,
+                connectPort: upstreamPort?.toString(),
                 clientAlpn: helloData.alpnProtocols,
                 ja3Fingerprint: calculateJa3FromFingerprintData(helloData.fingerprintData),
                 ja4Fingerprint: calculateJa4FromHelloData(helloData)
             };
 
-            if (shouldPassThrough(connectHostname, passThroughPatterns, interceptOnlyPatterns)) {
-                const upstreamPort = connectPort ? parseInt(connectPort, 10) : undefined;
-                passthroughListener(socket, connectHostname, upstreamPort);
+            if (shouldPassThrough(upstreamHostname, passThroughPatterns, interceptOnlyPatterns)) {
+                passthroughListener(socket, upstreamHostname, upstreamPort);
                 return; // Do not continue with TLS
             } else if (shouldPassThrough(sniHostname, passThroughPatterns, interceptOnlyPatterns)) {
                 passthroughListener(socket, sniHostname!); // Can't guess the port - not included in SNI