Add support for JSON-RPC specs in our public API list

Author: pimterry

src/components/settings/api-settings-card.tsx

@@ -24,6 +24,7 @@ import { SettingsButton, SettingsExplanation } from './settings-components';
 import { TextInput } from '../common/inputs';
 import { ApiStore } from '../../model/api/api-store';
 import { ContentLabel } from '../common/text-content';
+import { ApiMetadata } from '../../model/api/api-interfaces';
 
 const UploadSpecButton = styled(SettingsButton).attrs(() => ({
     type: 'submit'
@@ -278,7 +279,7 @@ export class ApiSettingsCard extends React.Component<
     saveApi = flow(function * (this: ApiSettingsCard) {
         const baseUrl = this.enteredBaseUrl.replace(/https?:\/\//, '');
 
-        const api = yield buildApiMetadataAsync(
+        const api: ApiMetadata = yield buildApiMetadataAsync(
             this.selectedSpec!,
             ['http://' + baseUrl, 'https://' + baseUrl]
         );

src/model/api/api-interfaces.ts

@@ -1,21 +1,23 @@
-import type * as querystring from 'querystring';
 import type {
+    OpenAPIObject,
     SchemaObject
 } from 'openapi-directory';
 
 import type {
     HtkResponse,
     Html
 } from "../../types";
-import { OpenApiMetadata } from './build-openapi';
+
+import type { OpenApiMetadata } from './build-api-metadata';
+import type { OpenRpcDocument, OpenRpcMetadata } from './jsonrpc';
 
 export type ApiMetadata =
-    | OpenApiMetadata;
+    | OpenApiMetadata
+    | OpenRpcMetadata;
 
-export interface ApiRequestMatcher {
-    pathMatcher: RegExp;
-    queryMatcher: querystring.ParsedUrlQuery;
-}
+export type ApiSpec =
+    | OpenAPIObject
+    | OpenRpcDocument;
 
 export interface ApiExchange {
     readonly service: ApiService;

src/model/api/api-store.ts

@@ -2,20 +2,20 @@ import * as _ from 'lodash';
 import { observable, observe, computed } from "mobx";
 import * as localForage from 'localforage';
 import * as serializr from 'serializr';
-import { findApi as findPublicOpenApi, OpenAPIObject } from 'openapi-directory';
+import { findApi as findPublicOpenApi } from 'openapi-directory';
 
 import { HtkRequest } from '../../types';
 import { reportError } from '../../errors';
 import { lazyObservablePromise } from "../../util/observable";
 import { hydrate, persist } from "../../util/mobx-persist/persist";
 
 import { AccountStore } from "../account/account-store";
-import { ApiMetadata } from "./api-interfaces";
+import { ApiMetadata, ApiSpec } from "./api-interfaces";
 import { buildApiMetadataAsync } from '../../services/ui-worker-api';
-import { findBestMatchingApi } from './openapi';
+import { matchOpenApiOperation } from './openapi';
 import { serializeRegex, serializeMap } from '../serialization';
 
-async function fetchApiMetadata(specId: string): Promise<OpenAPIObject> {
+async function fetchApiMetadata(specId: string): Promise<ApiSpec> {
     const specResponse = await fetch(`/api/${specId}.json`);
     return specResponse.json();
 }
@@ -125,12 +125,12 @@ export class ApiStore {
             : undefined;
     }
 
-    private publicOpenApiCache: _.Dictionary<Promise<ApiMetadata>> = {};
+    private publicApiCache: _.Dictionary<Promise<ApiMetadata>> = {};
 
     private getPublicApi(specId: string) {
-        const { publicOpenApiCache } = this;
-        if (!publicOpenApiCache[specId]) {
-            publicOpenApiCache[specId] = fetchApiMetadata(specId)
+        const { publicApiCache } = this;
+        if (!publicApiCache[specId]) {
+            publicApiCache[specId] = fetchApiMetadata(specId)
                 .then(buildApiMetadataAsync)
                 .catch((e) => {
                     console.log(`Failed to build API ${specId}`);
@@ -140,11 +140,11 @@ export class ApiStore {
                     throw e;
                 });
         }
-        return publicOpenApiCache[specId];
+        return publicApiCache[specId];
     }
 
     async getApi(request: HtkRequest): Promise<ApiMetadata | undefined> {
-        const { parsedUrl, url } = request;
+        const { parsedUrl } = request;
 
         // Is this a configured private API? I.e. has the user explicitly given
         // us a spec to use for requests like these.
@@ -154,10 +154,9 @@ export class ApiStore {
         }
 
         // If not, is this a known public API? (note that private always has precedence)
-        const requestUrl = `${parsedUrl.hostname}${parsedUrl.pathname}`;
+        const requestUrl = `${parsedUrl.host}${parsedUrl.pathname}`;
 
         let publicSpecId = findPublicOpenApi(requestUrl);
-
         if (!publicSpecId) return;
         if (!Array.isArray(publicSpecId)) publicSpecId = [publicSpecId];
 
@@ -179,4 +178,34 @@ export class ApiStore {
         return findBestMatchingApi(publicSpecs, request);
     }
 
+}
+
+// If we match multiple APIs, build all of them, try to match each one
+// against the request, and return only the matching one. This happens if
+// multiple services have the exact same base request URL (rds.amazonaws.com)
+export function findBestMatchingApi(
+    apis: ApiMetadata[],
+    request: HtkRequest
+): ApiMetadata | undefined {
+    const matchingApis = apis.filter((api) =>
+        api.type == 'openrpc' || // Since we have so few JSON-RPC APIs, we assume they match
+        matchOpenApiOperation(api, request).matched // For OpenAPI, we check for a matching op
+    );
+
+    // If we've successfully found one matching API, return it
+    if (matchingApis.length === 1) return matchingApis[0];
+
+    // If this is a non-matching request to one of these APIs, but we're not
+    // sure which, return the one with the most paths (as a best guess metric of
+    // which is the most popular service)
+    if (matchingApis.length === 0) return _.maxBy(apis, a => a.spec.paths.length)!;
+
+    // Otherwise we've matched multiple APIs which all define operations that
+    // could be this one request. Does exist right now (e.g. AWS RDS vs DocumentDB)
+
+    // Report this so we can try to improve & avoid in future.
+    reportError('Overlapping APIs', matchingApis);
+
+    // Return our guess of the most popular service, from the matching services only
+    return _.maxBy(matchingApis, a => a.spec.paths.length)!;
 }

src/model/api/build-openapi.ts renamed to src/model/api/build-api-metadata.ts

@@ -2,18 +2,23 @@ import * as _ from 'lodash';
 import * as querystring from 'querystring';
 
 import { OpenAPIObject, PathItemObject } from 'openapi-directory';
+import { MethodObject } from '@open-rpc/meta-schema';
 import * as Ajv from 'ajv';
 
-import type { ApiRequestMatcher } from './api-interfaces';
 import { openApiSchema } from './openapi-schema';
 import { dereference } from '../../util/json-schema';
-
+import { OpenRpcDocument, OpenRpcMetadata } from './jsonrpc';
 
 export interface OpenApiMetadata {
     type: 'openapi';
     spec: OpenAPIObject;
     serverMatcher: RegExp;
-    requestMatchers: Map<ApiRequestMatcher, Path>;
+    requestMatchers: Map<OpenApiRequestMatcher, Path>;
+}
+
+interface OpenApiRequestMatcher {
+    pathMatcher: RegExp;
+    queryMatcher: querystring.ParsedUrlQuery;
 }
 
 interface Path {
@@ -25,7 +30,9 @@ const filterSpec = new Ajv({
     removeAdditional: 'failing'
 }).compile(openApiSchema);
 
-function templateStringToRegexString(template: string): string {
+// Note that OpenAPI template strings are not the same as JSON template language templates,
+// which use ${...} instead of {...}.
+function openApiTemplateStringToRegexString(template: string): string {
     return template
         // Escape all regex chars *except* { }
         .replace(/[\^$\\.*+?()[\]|]/g, '\\$&')
@@ -35,7 +42,7 @@ function templateStringToRegexString(template: string): string {
         .replace(/\/$/, '');
 }
 
-export async function buildApiMetadata(
+export async function buildOpenApiMetadata(
     spec: OpenAPIObject,
     baseUrlOverrides?: string[]
 ): Promise<OpenApiMetadata> {
@@ -58,18 +65,18 @@ export async function buildApiMetadata(
     }
 
     // Now it's relatively small & tidy, dereference everything.
-    spec = <OpenAPIObject> dereference(spec);
+    spec = dereference(spec);
 
     const serverUrlRegexSources = baseUrlOverrides && baseUrlOverrides.length
         // Look for one of the given base URLs, ignoring trailing slashes
         ? baseUrlOverrides.map(url => _.escapeRegExp(url).replace(/\/$/, ''))
         // Look for any URL of the base servers in the spec
-        : spec.servers!.map(s => templateStringToRegexString(s.url));
+        : spec.servers!.map(s => openApiTemplateStringToRegexString(s.url));
 
     // Build a regex that matches any of these at the start of a URL
-    const serverMatcher = new RegExp(`^(${serverUrlRegexSources.join('|')})`, 'i')
+    const serverMatcher = new RegExp(`^(${serverUrlRegexSources.join('|')})`, 'i');
 
-    const requestMatchers = new Map<ApiRequestMatcher, Path>();
+    const requestMatchers = new Map<OpenApiRequestMatcher, Path>();
     _.entries(spec.paths)
         // Sort path & pathspec pairs to ensure that more specific paths are
         // always listed first, so that later on we can always use the first match
@@ -115,7 +122,7 @@ export async function buildApiMetadata(
             requestMatchers.set({
                 // Build a regex that matches this path on any of those base servers
                 pathMatcher: new RegExp(
-                    serverMatcher.source + templateStringToRegexString(realPath) + '/?$',
+                    serverMatcher.source + openApiTemplateStringToRegexString(realPath) + '/?$',
                     'i'
                 ),
                 // Some specs (AWS) also match requests by specific params
@@ -132,4 +139,38 @@ export async function buildApiMetadata(
         serverMatcher,
         requestMatchers
     };
+}
+
+// Approximately transform a JSON template string into a regex string that will match
+// values, using wildcards for each template. This is based on this draft RFC:
+// https://datatracker.ietf.org/doc/draft-jonas-json-template-language/
+// Only seems relevant to OpenRPC, doesn't seem widely used elsewhere.
+function jsonTemplateStringToRegexString(template: string): string {
+    return template
+        // Escape all regex chars *except* $, {, }
+        .replace(/[\^\\.*+?()[\]|]/g, '\\$&')
+        // Replace templates with wildcards
+        .replace(/\$\{([^/}]+)}/g, '([^\/]*)')
+        // Drop trailing slashes
+        .replace(/\/$/, '');
+}
+
+export function buildOpenRpcMetadata(spec: OpenRpcDocument, baseUrlOverrides?: string[]): OpenRpcMetadata {
+    spec = dereference(spec);
+
+    const serverUrlRegexSources = baseUrlOverrides && baseUrlOverrides.length
+        // Look for one of the given base URLs, ignoring trailing slashes
+        ? baseUrlOverrides.map(url => _.escapeRegExp(url).replace(/\/$/, ''))
+        // Look for any URL of the base servers in the spec
+        : spec.servers!.map(s => jsonTemplateStringToRegexString(s.url));
+
+    // Build a regex that matches any of these at the start of a URL
+    const serverMatcher = new RegExp(`^(${serverUrlRegexSources.join('|')})`, 'i');
+
+    return {
+        type: 'openrpc',
+        spec,
+        serverMatcher,
+        requestMatchers: _.keyBy(spec.methods, 'name') as _.Dictionary<MethodObject> // Dereferenced
+    };
 }

src/model/api/openapi.ts

@@ -29,43 +29,17 @@ import {
     ApiOperation,
     ApiRequest,
     ApiResponse,
-    ApiParameter
+    ApiParameter,
+    ApiMetadata
 } from './api-interfaces';
-import { OpenApiMetadata } from './build-openapi';
+import { OpenApiMetadata } from './build-api-metadata';
 import { fromMarkdown } from '../markdown';
 
 const paramValidator = new Ajv({
     coerceTypes: 'array',
     unknownFormats: 'ignore' // OpenAPI uses some non-standard formats
 });
 
-// If we match multiple APIs, build all of them, try to match each one
-// against the request, and return only the matching one. This happens if
-// multiple services have the exact same base request URL (rds.amazonaws.com)
-export function findBestMatchingApi(
-    apis: OpenApiMetadata[],
-    request: HtkRequest
-): OpenApiMetadata | undefined {
-    const matchingApis = apis.filter((api) => matchOperation(api, request).matched);
-
-    // If we've successfully found one matching API, return it
-    if (matchingApis.length === 1) return matchingApis[0];
-
-    // If this is a non-matching request to one of these APIs, but we're not
-    // sure which, return the one with the most paths (as a best guess metric of
-    // which is the most popular service)
-    if (matchingApis.length === 0) return _.maxBy(apis, a => a.spec.paths.length)!;
-
-    // Otherwise we've matched multiple APIs which all define operations that
-    // could be this one request. Does exist right now (e.g. AWS RDS vs DocumentDB)
-
-    // Report this so we can try to improve & avoid in future.
-    reportError('Overlapping APIs', matchingApis);
-
-    // Return our guess of the most popular service, from the matching services only
-    return _.maxBy(matchingApis, a => a.spec.paths.length)!;
-}
-
 function getPath(api: OpenApiMetadata, request: HtkRequest): {
     pathSpec: PathObject,
     path: string
@@ -308,7 +282,7 @@ export class OpenApiExchange implements ApiExchange {
         this.service = new OpenApiService(api.spec);
 
         this._spec = api.spec;
-        this._opSpec = matchOperation(api, request);
+        this._opSpec = matchOpenApiOperation(api, request);
 
         this.operation = new OpenApiOperation(this._opSpec);
         this.request = new OpenApiRequest(api.spec, this._opSpec, request);
@@ -342,7 +316,7 @@ class OpenApiService implements ApiService {
         const { info: service } = spec;
 
         this.name = service.title;
-        this.logoUrl = service['x-logo']?.url
+        this.logoUrl = service['x-logo']?.url;
         this.description = fromMarkdown(service.description);
         this.docsUrl = spec?.externalDocs?.url;
     }
@@ -353,7 +327,7 @@ class OpenApiService implements ApiService {
     public readonly docsUrl?: string;
 }
 
-function matchOperation(api: OpenApiMetadata, request: HtkRequest) {
+export function matchOpenApiOperation(api: OpenApiMetadata, request: HtkRequest) {
     const matchingPath = getPath(api, request);
 
     const { pathSpec, path } = matchingPath || {
@@ -376,7 +350,7 @@ function matchOperation(api: OpenApiMetadata, request: HtkRequest) {
     };
 }
 
-type MatchedOperation = ReturnType<typeof matchOperation>;
+type MatchedOperation = ReturnType<typeof matchOpenApiOperation>;
 
 class OpenApiOperation implements ApiOperation {
     constructor(

src/model/http/exchange.ts

@@ -22,6 +22,7 @@ import {
     asHeaderArray,
     lastHeader,
 } from '../../util';
+import { UnreachableCheck } from '../../util/error';
 import { lazyObservablePromise, ObservablePromise, observablePromise } from "../../util/observable";
 
 import { reportError } from '../../errors';
@@ -33,6 +34,7 @@ import { HTKEventBase } from '../events/event-base';
 import { ApiStore } from '../api/api-store';
 import { ApiExchange } from '../api/api-interfaces';
 import { OpenApiExchange } from '../api/openapi';
+import { parseRpcApiExchange } from '../api/jsonrpc';
 import { ApiMetadata } from '../api/api-interfaces';
 import { decodeBody } from '../../services/ui-worker-api';
 import {
@@ -326,7 +328,13 @@ export class HttpExchange extends HTKEventBase {
 
         if (apiMetadata) {
             try {
-                return new OpenApiExchange(apiMetadata, this);
+                if (apiMetadata.type === 'openapi') {
+                    return new OpenApiExchange(apiMetadata, this);
+                } else if (apiMetadata.type === 'openrpc') {
+                    return await parseRpcApiExchange(apiMetadata, this);
+                } else {
+                    throw new UnreachableCheck(apiMetadata, m => m.type);
+                }
             } catch (e) {
                 reportError(e);
                 throw e;