Move all body decoding to the server side in remote client setups

This is a major structural change, although it shouldn't result in any
visible changes in behaviour in most cases. Internally, this now decodes
all message bodies on the server side by default, so that client-side
decoding can be disabled entirely shortly (avoiding some awkward WASM
dependencies).

This does create some changes in the plugin model - while still
experimental, this is probably a breaking change there.

Author: pimterry

src/admin/admin-plugin-types.ts

@@ -19,6 +19,10 @@ export type PluginStartParams<Plugin> = Plugin extends AdminPlugin<infer StartPa
     ? StartParams
     : never;
 
+export type PluginStartDefaults<Plugins extends { [key: string]: AdminPlugin<any, any> }> = {
+    [key in keyof Plugins]?: Partial<PluginStartParams<Plugins[key]>>
+};
+
 export type PluginClientResponse<Plugin> = Plugin extends AdminPlugin<any, infer ClientResponse>
     ? ClientResponse
     : never;

src/admin/admin-server.ts

@@ -23,9 +23,8 @@ import { objectAllPromise } from '../util/promise';
 import { DEFAULT_ADMIN_SERVER_PORT } from '../types';
 
 import { RuleParameters } from '../rules/rule-parameters';
-import { AdminPlugin, PluginConstructorMap, PluginStartParamsMap } from './admin-plugin-types';
+import { AdminPlugin, PluginConstructorMap, PluginStartDefaults, PluginStartParamsMap } from './admin-plugin-types';
 import { parseAnyAst } from './graphql-utils';
-import { MockttpAdminPlugin } from './mockttp-admin-plugin';
 
 export interface AdminServerOptions<Plugins extends { [key: string]: AdminPlugin<any, any> }> {
     /**
@@ -53,7 +52,7 @@ export interface AdminServerOptions<Plugins extends { [key: string]: AdminPlugin
      * Override the default parameters for sessions started from this admin server. These values will be
      * used for each setting that is not explicitly specified by the client when creating a mock session.
      */
-    pluginDefaults?: Partial<PluginStartParamsMap<Plugins>>;
+    pluginDefaults?: PluginStartDefaults<Plugins>;
 
     /**
      * Some rule options can't easily be specified in remote clients, since they need to access
@@ -176,7 +175,7 @@ export class AdminServer<Plugins extends { [key: string]: AdminPlugin<any, any>
 
         this.app.use(bodyParser.json({ limit: '50mb' }));
 
-        const defaultPluginStartParams: Partial<PluginStartParamsMap<Plugins>> = options.pluginDefaults ?? {};
+        const defaultPluginStartParams: PluginStartDefaults<Plugins> = options.pluginDefaults ?? {};
 
         this.app.post('/start', async (req, res) => {
             try {

src/admin/mockttp-admin-model.ts

@@ -3,21 +3,29 @@ import { Duplex } from "stream";
 
 import { PubSub } from "graphql-subscriptions";
 import type { IResolvers } from "@graphql-tools/utils";
+import { ErrorLike, UnreachableCheck } from "@httptoolkit/util";
 
+import type { Headers } from '../types';
 import type { MockttpServer } from "../server/mockttp-server";
 import type { ServerMockedEndpoint } from "../server/mocked-endpoint";
 import type {
     MockedEndpoint,
     MockedEndpointData,
     CompletedRequest,
     CompletedResponse,
-    ClientError
+    ClientError,
+    CompletedBody
 } from "../types";
 import type { Serialized } from "../serialization/serialization";
 import type { RequestRuleData } from "../rules/requests/request-rule";
 import type { WebSocketRuleData } from "../rules/websockets/websocket-rule";
 
-import { deserializeRuleData, deserializeWebSocketRuleData } from "../rules/rule-deserialization";
+import {
+    deserializeRuleData,
+    deserializeWebSocketRuleData,
+    MockttpDeserializationOptions
+} from "../rules/rule-deserialization";
+import { decodeBodyBuffer } from "../util/request-utils";
 import { SubscribableEvent } from "../main";
 
 const graphqlSubscriptionPairs = Object.entries({
@@ -49,12 +57,51 @@ async function buildMockedEndpointData(endpoint: ServerMockedEndpoint): Promise<
     };
 }
 
+const decodeAndSerializeBody = async (body: CompletedBody, headers: Headers): Promise<
+    | false // Not required
+    | { decoded: Buffer, decodingError?: undefined } // Success
+    | { decodingError: string, decoded?: undefined } // Failure
+> => {
+    try {
+        const decoded = await decodeBodyBuffer(body.buffer, headers);
+        if (decoded === body.buffer) return false; // No decoding required - no-op.
+        else return { decoded }; // Successful decoding result
+    } catch (e) {
+        return { // Failed decoding - we just return the error message.
+            decodingError: (e as ErrorLike)?.message ?? 'Failed to decode message body'
+        };
+    }
+};
+
 export function buildAdminServerModel(
     mockServer: MockttpServer,
     stream: Duplex,
-    ruleParameters: { [key: string]: any }
+    ruleParams: { [key: string]: any },
+    options: {
+        messageBodyDecoding?: 'server-side' | 'none';
+    } = {}
 ): IResolvers {
     const pubsub = new PubSub();
+    const messageBodyDecoding = options.messageBodyDecoding || 'server-side';
+
+    const ruleDeserializationOptions: MockttpDeserializationOptions = {
+        bodySerializer: messageBodyDecoding === 'server-side'
+            ? async (body, headers) => {
+                const encoded = body.buffer.toString('base64');
+                const result = await decodeAndSerializeBody(body, headers);
+                if (result === false) { // No decoding required - no-op.
+                    return { encoded };
+                } else if (result.decodingError !== undefined) { // Failed decoding - we just return the error message.
+                    return { encoded, decodingError: result.decodingError };
+                } else if (result.decoded) { // Success - we return both formats to the client
+                    return { encoded, decoded: result.decoded.toString('base64') };
+                } else {
+                    throw new UnreachableCheck(result);
+                }
+            }
+            : (body) => body.buffer.toString('base64'), // 'None' = just send encoded body (as base64).
+        ruleParams
+    };
 
     for (let [gqlName, eventName] of graphqlSubscriptionPairs) {
         mockServer.on(eventName as any, (evt) => {
@@ -91,30 +138,30 @@ export function buildAdminServerModel(
 
         Mutation: {
             addRule: async (__: any, { input }: { input: Serialized<RequestRuleData> }) => {
-                return mockServer.addRequestRule(deserializeRuleData(input, stream, ruleParameters));
+                return mockServer.addRequestRule(deserializeRuleData(input, stream, ruleDeserializationOptions));
             },
             addRules: async (__: any, { input }: { input: Array<Serialized<RequestRuleData>> }) => {
                 return mockServer.addRequestRules(...input.map((rule) =>
-                    deserializeRuleData(rule, stream, ruleParameters)
+                    deserializeRuleData(rule, stream, ruleDeserializationOptions)
                 ));
             },
             setRules: async (__: any, { input }: { input: Array<Serialized<RequestRuleData>> }) => {
                 return mockServer.setRequestRules(...input.map((rule) =>
-                    deserializeRuleData(rule, stream, ruleParameters)
+                    deserializeRuleData(rule, stream, ruleDeserializationOptions)
                 ));
             },
 
             addWebSocketRule: async (__: any, { input }: { input: Serialized<WebSocketRuleData> }) => {
-                return mockServer.addWebSocketRule(deserializeWebSocketRuleData(input, stream, ruleParameters));
+                return mockServer.addWebSocketRule(deserializeWebSocketRuleData(input, stream, ruleDeserializationOptions));
             },
             addWebSocketRules: async (__: any, { input }: { input: Array<Serialized<WebSocketRuleData>> }) => {
                 return mockServer.addWebSocketRules(...input.map((rule) =>
-                    deserializeWebSocketRuleData(rule, stream, ruleParameters)
+                    deserializeWebSocketRuleData(rule, stream, ruleDeserializationOptions)
                 ));
             },
             setWebSocketRules: async (__: any, { input }: { input: Array<Serialized<WebSocketRuleData>> }) => {
                 return mockServer.setWebSocketRules(...input.map((rule) =>
-                    deserializeWebSocketRuleData(rule, stream, ruleParameters)
+                    deserializeWebSocketRuleData(rule, stream, ruleDeserializationOptions)
                 ));
             }
         },
@@ -124,12 +171,20 @@ export function buildAdminServerModel(
         Request: {
             body: (request: CompletedRequest) => {
                 return request.body.buffer;
+            },
+            decodedBody: async (request: CompletedRequest) => {
+                return (await decodeAndSerializeBody(request.body, request.headers))
+                    || {}; // No decoding required
             }
         },
 
         Response: {
             body: (response: CompletedResponse) => {
                 return response.body.buffer;
+            },
+            decodedBody: async (response: CompletedResponse) => {
+                return (await decodeAndSerializeBody(response.body, response.headers))
+                    || {}; // No decoding required
             }
         },
 

src/admin/mockttp-admin-plugin.ts

@@ -10,6 +10,7 @@ import { MockttpSchema } from './mockttp-schema';
 
 export interface MockttpPluginOptions {
     options?: Partial<MockttpOptions>;
+    messageBodyDecoding: 'server-side' | 'none';
     port?: number | PortRange;
 }
 
@@ -24,9 +25,13 @@ export class MockttpAdminPlugin implements AdminPlugin<
 > {
 
     private mockServer!: MockttpServer;
+    private messageBodyDecoding!: 'server-side' | 'none';
 
-    async start({ port, options }: MockttpPluginOptions) {
+    async start({ port, options, messageBodyDecoding }: MockttpPluginOptions) {
         this.mockServer = new MockttpServer(options);
+        this.messageBodyDecoding = messageBodyDecoding ||
+            'none'; // Backward compat - clients that don't set this option expect 'none'.
+
         await this.mockServer.start(port);
 
         return {
@@ -54,6 +59,8 @@ export class MockttpAdminPlugin implements AdminPlugin<
     schema = MockttpSchema;
 
     buildResolvers(stream: Duplex, ruleParameters: { [key: string]: any }) {
-        return buildAdminServerModel(this.mockServer, stream, ruleParameters)
+        return buildAdminServerModel(this.mockServer, stream, ruleParameters, {
+            messageBodyDecoding: this.messageBodyDecoding
+        })
     };
 }

src/admin/mockttp-schema.ts

@@ -176,6 +176,8 @@ export const MockttpSchema = gql`
         rawHeaders: Json!
 
         body: Buffer!
+        decodedBody: DecodingResult!
+
         rawTrailers: Json!
     }
 
@@ -212,7 +214,10 @@ export const MockttpSchema = gql`
 
         headers: Json!
         rawHeaders: Json!
+
         body: Buffer!
+        decodedBody: DecodingResult!
+
         rawTrailers: Json!
     }
 
@@ -241,4 +246,9 @@ export const MockttpSchema = gql`
         hostname: String!
         port: Int!
     }
+
+    type DecodingResult {
+        decoded: Buffer
+        decodingError: String
+    }
 `;

src/client/mockttp-admin-request-builder.ts

@@ -4,8 +4,7 @@ import gql from 'graphql-tag';
 
 import { MockedEndpoint, MockedEndpointData } from "../types";
 
-import { buildBodyReader } from '../util/request-utils';
-import { objectHeadersToRaw, rawHeadersToObject } from '../util/header-utils';
+import { rawHeadersToObject } from '../util/header-utils';
 
 import { AdminQuery } from './admin-query';
 import { SchemaIntrospector } from './schema-introspection';
@@ -17,6 +16,7 @@ import { SubscribableEvent } from '../mockttp';
 import { MockedEndpointClient } from "./mocked-endpoint-client";
 import { AdminClient } from './admin-client';
 import { serializeRuleData } from '../rules/rule-serialization';
+import { deserializeBodyReader } from '../serialization/body-serialization';
 
 function normalizeHttpMessage(message: any, event?: SubscribableEvent) {
     if (message.timingEvents) {
@@ -43,9 +43,18 @@ function normalizeHttpMessage(message: any, event?: SubscribableEvent) {
     }
 
     if (message.body !== undefined) {
-        // Body is serialized as the raw encoded buffer in base64
-        message.body = buildBodyReader(Buffer.from(message.body, 'base64'), message.headers);
+        // This will be unset if a) no decoding is required (so message.body is already decoded implicitly),
+        // b) if messageBodyDecoding is set to 'none', or c) if the server is <v4 and doesn't do decoding.
+        let { decoded, decodingError } = message.decodedBody || {};
+
+        message.body = deserializeBodyReader(
+            message.body,
+            decoded,
+            decodingError,
+            message.headers
+        );
     }
+    delete message.decodedBody;
 
     // For backwards compat, all except errors should have tags if they're missing
     if (!message.tags) message.tags = [];
@@ -77,9 +86,14 @@ function normalizeWebSocketMessage(message: any) {
  */
 export class MockttpAdminRequestBuilder {
 
+    private messageBodyDecoding: 'server-side' | 'none';
+
     constructor(
-        private schema: SchemaIntrospector
-    ) {}
+        private schema: SchemaIntrospector,
+        options: { messageBodyDecoding: 'server-side' | 'none' } = { messageBodyDecoding: 'server-side' }
+    ) {
+        this.messageBodyDecoding = options.messageBodyDecoding;
+    }
 
     buildAddRequestRulesQuery(
         rules: Array<RequestRuleData>,
@@ -259,6 +273,10 @@ export class MockttpAdminRequestBuilder {
 
                     rawHeaders
                     body
+                    ${this.schema.typeHasField('Request', 'decodedBody') && this.messageBodyDecoding === 'server-side'
+                        ? 'decodedBody { decoded, decodingError }'
+                        : ''
+                    }
                     ${this.schema.asOptionalField('Request', 'rawTrailers')}
 
                     timingEvents
@@ -274,6 +292,10 @@ export class MockttpAdminRequestBuilder {
 
                     rawHeaders
                     body
+                    ${this.schema.typeHasField('Response', 'decodedBody') && this.messageBodyDecoding === 'server-side'
+                        ? 'decodedBody { decoded, decodingError }'
+                        : ''
+                    }
                     ${this.schema.asOptionalField('Response', 'rawTrailers')}
 
                     timingEvents
@@ -298,6 +320,10 @@ export class MockttpAdminRequestBuilder {
 
                     rawHeaders
                     body
+                    ${this.schema.typeHasField('Request', 'decodedBody') && this.messageBodyDecoding === 'server-side'
+                        ? 'decodedBody { decoded, decodingError }'
+                        : ''
+                    }
                     ${this.schema.asOptionalField('Request', 'rawTrailers')}
 
                     timingEvents
@@ -313,6 +339,10 @@ export class MockttpAdminRequestBuilder {
 
                     rawHeaders
                     body
+                    ${this.schema.typeHasField('Response', 'decodedBody') && this.messageBodyDecoding === 'server-side'
+                        ? 'decodedBody { decoded, decodingError }'
+                        : ''
+                    }
                     ${this.schema.asOptionalField('Response', 'rawTrailers')}
 
                     timingEvents
@@ -460,6 +490,11 @@ export class MockttpAdminRequestBuilder {
                         rawHeaders
 
                         body
+                        ${this.schema.typeHasField('Response', 'decodedBody') && this.messageBodyDecoding === 'server-side'
+                            ? 'decodedBody { decoded, decodingError }'
+                            : ''
+                        }
+
                         ${this.schema.asOptionalField('Response', 'rawTrailers')}
                     }
                 }
@@ -563,7 +598,11 @@ export class MockttpAdminRequestBuilder {
 
                                 rawHeaders
 
-                                body,
+                                body
+                                ${this.schema.typeHasField('Request', 'decodedBody') && this.messageBodyDecoding === 'server-side'
+                                    ? 'decodedBody { decoded, decodingError }'
+                                    : ''
+                                }
                                 timingEvents
                                 httpVersion
                             }