fix (cli): Parse indented code blocks

packages/cli/src/rpc/navie/models/providers/openai.ts

@@ -11,6 +11,9 @@ interface OpenAiModelResponse {
   }[];
 }
 
+const EXCLUDED_MODELS_REGEX =
+  /preview|embedding|tts|moderation|dall-e|chatgpt|transcribe|image|\d{3,}/g;
+
 export async function fetchOpenAIModels(apiKey: string): Promise<NavieRpc.V1.Models.Model[]> {
   const models: NavieRpc.V1.Models.Model[] = [];
   try {
@@ -29,8 +32,7 @@ export async function fetchOpenAIModels(apiKey: string): Promise<NavieRpc.V1.Mod
       !['openai-internal', 'openai'].includes(model.owned_by);
 
     const isRelevantModel = (model: OpenAiModelResponse['data'][number]) =>
-      model.owned_by !== 'system' ||
-      !model.id.match(/preview|embedding|tts|moderation|dall-e|chatgpt|transcribe|\d{3,}/g);
+      model.owned_by !== 'system' || !model.id.match(EXCLUDED_MODELS_REGEX);
 
     const json = (await res.json()) as OpenAiModelResponse;
     models.push(

packages/cli/src/rpc/navie/thread/index.ts

@@ -224,11 +224,11 @@ export class Thread {
    * @param messageId The message id associated with the token
    */
   private onToken(token: string, messageId: string) {
-    const subTokens = token.split(/^(`{3,})\n?/gm);
+    const subTokens = token.split(/^(\s*?`{3,})/gm);
     for (let subToken of subTokens) {
       if (subToken.length === 0) continue;
 
-      const fileMatch = subToken.match(/^<!-- file: (.*) -->\s*?\n?/m);
+      const fileMatch = subToken.match(/^\s*?<!-- file: (.*) -->\s*?\n?/m);
       if (fileMatch) {
         // TODO: Should this be a file URI? We don't currently include line ranges.
         this.codeBlockUri = this.codeBlockUri ?? URI.random().toString();
@@ -260,7 +260,7 @@ export class Thread {
       this.lastTokenBeganCodeBlock = false;
 
       let clearCodeBlock = false;
-      if (subToken.match(/^`{3,}/)) {
+      if (subToken.match(/^\s*?`{3,}/)) {
         // Code block fences
         if (this.codeBlockLength === undefined) {
           this.codeBlockUri = this.codeBlockUri ?? URI.random().toString();

packages/cli/tests/unit/rpc/navie/thread/index.spec.ts

@@ -514,6 +514,116 @@ describe('Thread', () => {
         },
       ]);
     });
+    it('ignores whitespace leading file comment directives', () => {
+      const listener = jest.fn();
+      thread.on('event', 'test-client', listener);
+      thread.onToken('   <!-- file: test-file.md -->', 'test-message-id');
+      expect(listener.mock.calls.flat()).toStrictEqual([
+        {
+          type: 'token-metadata',
+          codeBlockUri: expect.any(String),
+          metadata: {
+            location: 'test-file.md',
+          },
+          time: expect.any(Number),
+        },
+      ]);
+    });
+    it('ignores whitespace leading code fences', () => {
+      const listener = jest.fn();
+      const token = `1. Python
+    \`\`\`python
+    for i in range(1, 11):
+        print(i)
+    \`\`\`
+2. JavaScript
+    \`\`\`js
+    for (let i = 1; i <= 10; i++) {
+      console.log(i);
+    }
+    \`\`\`
+`;
+      thread.on('event', 'test-client', listener);
+      thread.onToken(token, 'test-message-id');
+      expect(listener.mock.calls.flat()).toStrictEqual([
+        {
+          type: 'token',
+          token: '1. Python\n',
+          messageId: 'test-message-id',
+          time: expect.any(Number),
+        },
+        {
+          type: 'token',
+          token: '    ```',
+          messageId: 'test-message-id',
+          time: expect.any(Number),
+          codeBlockUri: expect.any(String),
+        },
+        {
+          type: 'token-metadata',
+          codeBlockUri: expect.any(String),
+          metadata: {
+            language: 'python',
+          },
+          time: expect.any(Number),
+        },
+        {
+          type: 'token',
+          token: 'python\n    for i in range(1, 11):\n        print(i)\n',
+          messageId: 'test-message-id',
+          time: expect.any(Number),
+          codeBlockUri: expect.any(String),
+        },
+        {
+          type: 'token',
+          token: '    ```',
+          messageId: 'test-message-id',
+          time: expect.any(Number),
+          codeBlockUri: expect.any(String),
+        },
+        {
+          type: 'token',
+          token: '\n2. JavaScript\n',
+          messageId: 'test-message-id',
+          time: expect.any(Number),
+        },
+        {
+          type: 'token',
+          token: '    ```',
+          messageId: 'test-message-id',
+          time: expect.any(Number),
+          codeBlockUri: expect.any(String),
+        },
+        {
+          type: 'token-metadata',
+          codeBlockUri: expect.any(String),
+          metadata: {
+            language: 'js',
+          },
+          time: expect.any(Number),
+        },
+        {
+          type: 'token',
+          token: 'js\n    for (let i = 1; i <= 10; i++) {\n      console.log(i);\n    }\n',
+          messageId: 'test-message-id',
+          time: expect.any(Number),
+          codeBlockUri: expect.any(String),
+        },
+        {
+          type: 'token',
+          token: '    ```',
+          messageId: 'test-message-id',
+          time: expect.any(Number),
+          codeBlockUri: expect.any(String),
+        },
+        {
+          type: 'token',
+          token: '\n',
+          messageId: 'test-message-id',
+          time: expect.any(Number),
+        },
+      ]);
+    });
   });
 
   describe('sendMessage', () => {