diff --git a/packages/cli/src/rpc/navie/models/providers/openai.ts b/packages/cli/src/rpc/navie/models/providers/openai.ts index 7bed3198ce..fb6943be76 100644 --- a/packages/cli/src/rpc/navie/models/providers/openai.ts +++ b/packages/cli/src/rpc/navie/models/providers/openai.ts @@ -11,6 +11,9 @@ interface OpenAiModelResponse { }[]; } +const EXCLUDED_MODELS_REGEX = + /preview|embedding|tts|moderation|dall-e|chatgpt|transcribe|image|\d{3,}/g; + export async function fetchOpenAIModels(apiKey: string): Promise { const models: NavieRpc.V1.Models.Model[] = []; try { @@ -29,8 +32,7 @@ export async function fetchOpenAIModels(apiKey: string): Promise - model.owned_by !== 'system' || - !model.id.match(/preview|embedding|tts|moderation|dall-e|chatgpt|transcribe|\d{3,}/g); + model.owned_by !== 'system' || !model.id.match(EXCLUDED_MODELS_REGEX); const json = (await res.json()) as OpenAiModelResponse; models.push( diff --git a/packages/cli/src/rpc/navie/thread/index.ts b/packages/cli/src/rpc/navie/thread/index.ts index c46e55af4d..83981b447d 100644 --- a/packages/cli/src/rpc/navie/thread/index.ts +++ b/packages/cli/src/rpc/navie/thread/index.ts @@ -224,11 +224,11 @@ export class Thread { * @param messageId The message id associated with the token */ private onToken(token: string, messageId: string) { - const subTokens = token.split(/^(`{3,})\n?/gm); + const subTokens = token.split(/^(\s*?`{3,})/gm); for (let subToken of subTokens) { if (subToken.length === 0) continue; - const fileMatch = subToken.match(/^\s*?\n?/m); + const fileMatch = subToken.match(/^\s*?\s*?\n?/m); if (fileMatch) { // TODO: Should this be a file URI? We don't currently include line ranges. this.codeBlockUri = this.codeBlockUri ?? URI.random().toString(); @@ -260,7 +260,7 @@ export class Thread { this.lastTokenBeganCodeBlock = false; let clearCodeBlock = false; - if (subToken.match(/^`{3,}/)) { + if (subToken.match(/^\s*?`{3,}/)) { // Code block fences if (this.codeBlockLength === undefined) { this.codeBlockUri = this.codeBlockUri ?? URI.random().toString(); diff --git a/packages/cli/tests/unit/rpc/navie/thread/index.spec.ts b/packages/cli/tests/unit/rpc/navie/thread/index.spec.ts index d1fb199cb5..407a6b87b9 100644 --- a/packages/cli/tests/unit/rpc/navie/thread/index.spec.ts +++ b/packages/cli/tests/unit/rpc/navie/thread/index.spec.ts @@ -514,6 +514,116 @@ describe('Thread', () => { }, ]); }); + it('ignores whitespace leading file comment directives', () => { + const listener = jest.fn(); + thread.on('event', 'test-client', listener); + thread.onToken(' ', 'test-message-id'); + expect(listener.mock.calls.flat()).toStrictEqual([ + { + type: 'token-metadata', + codeBlockUri: expect.any(String), + metadata: { + location: 'test-file.md', + }, + time: expect.any(Number), + }, + ]); + }); + it('ignores whitespace leading code fences', () => { + const listener = jest.fn(); + const token = `1. Python + \`\`\`python + for i in range(1, 11): + print(i) + \`\`\` +2. JavaScript + \`\`\`js + for (let i = 1; i <= 10; i++) { + console.log(i); + } + \`\`\` +`; + thread.on('event', 'test-client', listener); + thread.onToken(token, 'test-message-id'); + expect(listener.mock.calls.flat()).toStrictEqual([ + { + type: 'token', + token: '1. Python\n', + messageId: 'test-message-id', + time: expect.any(Number), + }, + { + type: 'token', + token: ' ```', + messageId: 'test-message-id', + time: expect.any(Number), + codeBlockUri: expect.any(String), + }, + { + type: 'token-metadata', + codeBlockUri: expect.any(String), + metadata: { + language: 'python', + }, + time: expect.any(Number), + }, + { + type: 'token', + token: 'python\n for i in range(1, 11):\n print(i)\n', + messageId: 'test-message-id', + time: expect.any(Number), + codeBlockUri: expect.any(String), + }, + { + type: 'token', + token: ' ```', + messageId: 'test-message-id', + time: expect.any(Number), + codeBlockUri: expect.any(String), + }, + { + type: 'token', + token: '\n2. JavaScript\n', + messageId: 'test-message-id', + time: expect.any(Number), + }, + { + type: 'token', + token: ' ```', + messageId: 'test-message-id', + time: expect.any(Number), + codeBlockUri: expect.any(String), + }, + { + type: 'token-metadata', + codeBlockUri: expect.any(String), + metadata: { + language: 'js', + }, + time: expect.any(Number), + }, + { + type: 'token', + token: 'js\n for (let i = 1; i <= 10; i++) {\n console.log(i);\n }\n', + messageId: 'test-message-id', + time: expect.any(Number), + codeBlockUri: expect.any(String), + }, + { + type: 'token', + token: ' ```', + messageId: 'test-message-id', + time: expect.any(Number), + codeBlockUri: expect.any(String), + }, + { + type: 'token', + token: '\n', + messageId: 'test-message-id', + time: expect.any(Number), + }, + ]); + }); }); describe('sendMessage', () => {