Finish passthrough option support for the Send API

This adds upstream proxy & DNS support, allowing for routing that
matches our intercepted Docker setup. Should mean we're now feature
complete for the core "accurately resend requests" workflow, woo!

Each of these has been manually tested with a hacked up UI demo and
everything seems to work correctly, so I think we're good to go...

Author: pimterry

package-lock.json

@@ -76,7 +76,7 @@
     "mime-types": "^2.1.27",
     "mobx": "^6.3.5",
     "mockrtc": "^0.3.1",
-    "mockttp": "^3.7.6",
+    "mockttp": "^3.8.0",
     "node-abort-controller": "^3.0.1",
     "node-fetch": "^2.6.1",
     "node-forge": "^1.3.0",

package.json

@@ -1,5 +1,4 @@
 import * as _ from 'lodash';
-import * as stream from 'stream';
 import * as os from 'os';
 
 import { generateSPKIFingerprint } from 'mockttp';
@@ -11,7 +10,8 @@ import { reportError, addBreadcrumb } from '../error-tracking';
 import { HtkConfig } from "../config";
 import { ActivationError, Interceptor } from "../interceptors";
 import { getDnsServer } from '../dns-server';
-import * as Client from '../client/client';
+import * as Client from '../client/client-types';
+import { HttpClient } from '../client/http-client';
 
 const INTERCEPTOR_TIMEOUT = 1000;
 
@@ -21,6 +21,7 @@ export class ApiModel {
         private config: HtkConfig,
         private interceptors: _.Dictionary<Interceptor>,
         private getRuleParamKeys: () => string[],
+        private httpClient: HttpClient,
         private callbacks: {
             onTriggerUpdate: () => void,
             onTriggerShutdown: () => void
@@ -209,7 +210,7 @@ export class ApiModel {
         requestDefinition: Client.RequestDefinition,
         requestOptions: Client.RequestOptions
     ) {
-        return Client.sendRequest(requestDefinition, requestOptions);
+        return this.httpClient.sendRequest(requestDefinition, requestOptions);
     }
 
 }

src/api/api-model.ts

@@ -12,6 +12,7 @@ import { shutdown } from '../shutdown';
 import { ApiModel } from './api-model';
 import { exposeGraphQLAPI } from './graphql-api';
 import { exposeRestAPI } from './rest-api';
+import { HttpClient } from '../client/http-client';
 
 /**
  * This file contains the core server API, used by the UI to query
@@ -39,7 +40,11 @@ export class HttpToolkitServerApi extends events.EventEmitter {
 
     private server: express.Application;
 
-    constructor(config: HtkConfig, getRuleParamKeys: () => string[]) {
+    constructor(
+        config: HtkConfig,
+        httpClient: HttpClient,
+        getRuleParamKeys: () => string[]
+    ) {
         super();
 
         const interceptors = buildInterceptors(config);
@@ -114,6 +119,7 @@ export class HttpToolkitServerApi extends events.EventEmitter {
             config,
             interceptors,
             getRuleParamKeys,
+            httpClient,
             {
                 onTriggerUpdate: () => this.emit('update-requested'),
                 onTriggerShutdown: () => shutdown('API call')

src/api/api-server.ts

@@ -14,7 +14,7 @@ import type { ParsedQs } from 'qs';
 import { ErrorLike, StatusError } from '../util/error';
 import { reportError } from '../error-tracking';
 import { ApiModel } from './api-model';
-import * as Client from '../client/client';
+import * as Client from '../client/client-types';
 
 /**
  * This file exposes the API model via a REST-ish classic HTTP API.
@@ -96,18 +96,13 @@ export function exposeRestAPI(
 
         // Various buffers are serialized as base64 (for both requests & responses)
         request.rawBody = Buffer.from(request.rawBody ?? '', 'base64');
-        if (options.trustAdditionalCAs) {
-            options.trustAdditionalCAs = options.trustAdditionalCAs.map(
-                ({ cert }: { cert: string }) => Buffer.from(cert, 'base64')
-            );
-        }
         if (options.clientCertificate) {
             options.clientCertificate.pfx = Buffer.from(options.clientCertificate.pfx, 'base64');
         }
 
         // Start actually sending the request:
         const abortController = new AbortController();
-        const resultStream = apiModel.sendRequest(request, {
+        const resultStream = await apiModel.sendRequest(request, {
             ...options,
             abortSignal: abortController.signal
         });

src/api/rest-api.ts

@@ -0,0 +1,116 @@
+import type * as Mockttp from 'mockttp';
+
+// --- Request definition types ---
+
+export type RawHeaders = Mockttp.RawHeaders;
+
+export interface RequestDefinition {
+    method: string;
+    url: string;
+
+    /**
+     * The raw headers to send. These will be sent exactly as provided - no headers
+     * will be added automatically.
+     *
+     * Note that this means omitting the 'Host' header may cause problems, as will
+     * omitting both 'Content-Length' and 'Transfer-Encoding' on requests with
+     * bodies.
+     */
+    headers: RawHeaders;
+
+    rawBody?: Uint8Array;
+}
+
+// --- Request option types ---
+
+export const RULE_PARAM_REF_KEY = '__rule_param_reference__';
+type ClientProxyRuleParamReference = { [RULE_PARAM_REF_KEY]: string };
+
+export type ClientProxyConfig =
+    | undefined // No Docker, no user or system proxy
+    | Mockttp.ProxySetting // User or system proxy
+    | ClientProxyRuleParamReference // Docker proxy (must be dereferenced)
+    | Array<Mockttp.ProxySetting | ClientProxyRuleParamReference> // Both, ordered
+
+export interface RequestOptions {
+    /**
+     * A list of hostnames for which server certificate and TLS version errors
+     * should be ignored (none, by default).
+     *
+     * If set to 'true', HTTPS errors will be ignored for all hosts. WARNING:
+     * Use this at your own risk. Setting this to `true` can open your
+     * application to MITM attacks and should never be used over any network
+     * that is not completed trusted end-to-end.
+     */
+    ignoreHostHttpsErrors?: string[] | boolean;
+
+    /**
+     * An array of additional certificates, which should be trusted as certificate
+     * authorities for upstream hosts, in addition to Node.js's built-in certificate
+     * authorities.
+     *
+     * Each certificate should be an object with a `cert` key containing the PEM
+     * certificate as a string.
+     */
+    trustAdditionalCAs?: Array<{ cert: string }>;
+
+    /**
+     * A client certificate that should be used for the connection, if the server
+     * requests one during the TLS handshake.
+     */
+    clientCertificate?: { pfx: Buffer, passphrase?: string };
+
+    /**
+     * Proxy configuration, specifying how (if at all) a proxy that should be used
+     * for upstream connections.
+     */
+    proxyConfig?: ClientProxyConfig;
+
+    /**
+     * Custom DNS options, to allow configuration of the resolver used when
+     * forwarding requests upstream. Passing any option switches from using node's
+     * default dns.lookup function to using the cacheable-lookup module, which
+     * will cache responses.
+     */
+    lookupOptions?: { servers?: string[] };
+
+    /**
+     * An abort signal, which can be used to cancel the in-process request if
+     * required.
+     */
+    abortSignal?: AbortSignal;
+}
+
+// --- Response types ---
+
+export type ResponseStreamEvents =
+    | RequestStart
+    | ResponseHead
+    | ResponseBodyPart
+    | ResponseEnd;
+// Other notable event is errors (via 'error' event)
+
+export interface RequestStart {
+    type: 'request-start';
+    startTime: number; // Unix timestamp
+    timestamp: number; // High precision timer (for relative calculations on later events)
+}
+
+export interface ResponseHead {
+    type: 'response-head';
+    statusCode: number;
+    statusMessage?: string;
+    headers: RawHeaders;
+    timestamp: number;
+}
+
+export interface ResponseBodyPart {
+    type: 'response-body-part';
+    rawBody: Buffer;
+    timestamp: number;
+}
+
+export interface ResponseEnd {
+    type: 'response-end';
+    timestamp: number;
+}