Cody: Add e2e test for inline assist and add cody tests to sg(#52337)

Author: abeatrix

client/cody/CONTRIBUTING.md

@@ -32,15 +32,40 @@
 1. Unit tests:
 
    ```shell
-   $ cd client/cody
-   $ pnpm test:unit
+   cd client/cody
+   pnpm test:unit
    ```
 
 2. Integration tests:
 
    ```shell
-   $ cd client/cody
-   $ pnpm test:integration
+   cd client/cody
+   pnpm test:integration
+   ```
+
+3. E2E tests:
+
+   To run all the tests inside the `client/cody/test/e2e` directory:
+
+   ```shell
+   cd client/cody
+   pnpm test:e2e
+   ```
+
+   To run test individually, pass in the name of the test by replacing $TEST_NAME below.
+
+   ```sh
+   pnpm test:e2e $TEST_NAME
+   # Example: Run the inline-assist test only
+   pnpm test:e2e inline-assist
+   ```
+
+   Example: Run the inline-assist test only
+
+   ```sh
+   pnpm test:e2e --debug
+   # Example: Run the inline-assist test in debug mode
+   pnpm test:e2e inline-assist --debug
    ```
 
 ## Release Process

client/cody/package.json

@@ -59,7 +59,7 @@
     "lint:js": "eslint --cache '**/*.[tj]s?(x)'",
     "test:integration": "tsc -b ./test/integration && pnpm run --silent build:dev && node --inspect -r ts-node/register out/test/integration/main.js",
     "test:unit": "jest",
-    "test:e2e": "npx playwright install && pnpm run --silent build:dev && playwright test test/e2e",
+    "test:e2e": "npx playwright install && pnpm run --silent build:dev && playwright test",
     "vscode:prepublish": "scripts/check-rg.sh",
     "vsce:package": "pnpm --silent build && vsce package --no-dependencies -o dist/cody.vsix",
     "vsce:prerelease": "pnpm --silent build && vsce package patch --pre-release --no-dependencies -o dist/cody.vsix",

client/cody/playwright.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from '@playwright/test'
+
+// eslint-disable-next-line import/no-default-export
+export default defineConfig({
+    workers: 1,
+    testDir: 'test/e2e',
+})

client/cody/src/configuration.ts

@@ -12,6 +12,8 @@ import { SecretStorage, getAccessToken } from './services/SecretStorageProvider'
  * All configuration values, with some sanitization performed.
  */
 export function getConfiguration(config: Pick<vscode.WorkspaceConfiguration, 'get'>): Configuration {
+    const isTesting = process.env.CODY_TESTING === 'true'
+
     let debugRegex: RegExp | null = null
     try {
         const debugPattern: string | null = config.get<string | null>('cody.debug.filter', null)
@@ -30,15 +32,15 @@ export function getConfiguration(config: Pick<vscode.WorkspaceConfiguration, 'ge
     return {
         serverEndpoint: sanitizeServerEndpoint(config.get('cody.serverEndpoint', '')),
         codebase: sanitizeCodebase(config.get('cody.codebase')),
+        customHeaders: config.get<object>('cody.customHeaders', {}) as Record<string, string>,
+        useContext: config.get<ConfigurationUseContext>('cody.useContext') || 'embeddings',
         debugEnable: config.get<boolean>('cody.debug.enable', false),
         debugVerbose: config.get<boolean>('cody.debug.verbose', false),
         debugFilter: debugRegex,
-        useContext: config.get<ConfigurationUseContext>('cody.useContext') || 'embeddings',
-        experimentalSuggest: config.get('cody.experimental.suggestions', false),
-        experimentalChatPredictions: config.get('cody.experimental.chatPredictions', false),
-        experimentalInline: config.get('cody.experimental.inline', false),
-        experimentalGuardrails: config.get('cody.experimental.guardrails', false),
-        customHeaders: config.get<object>('cody.customHeaders', {}) as Record<string, string>,
+        experimentalSuggest: config.get('cody.experimental.suggestions', isTesting),
+        experimentalChatPredictions: config.get('cody.experimental.chatPredictions', isTesting),
+        experimentalInline: config.get('cody.experimental.inline', isTesting),
+        experimentalGuardrails: config.get('cody.experimental.guardrails', isTesting),
     }
 }
 

client/cody/src/services/DecorationProvider.ts

@@ -47,7 +47,7 @@ export class DecorationProvider {
         await vscode.window.showTextDocument(this.fileUri)
         if (this.status === CodyTaskState.done) {
             this.decorationTypePending.dispose()
-            this.decorations.push({ range, hoverMessage: 'Cody Task#' + this.id })
+            this.decorations.push({ range, hoverMessage: 'Cody Assist #' + this.id })
             this.decorationsForIcon.push({ range: getSingleLineRange(range.start.line) })
             vscode.window.activeTextEditor?.setDecorations(this.decorationTypeIcon, this.decorationsForIcon)
             vscode.window.activeTextEditor?.setDecorations(this.decorationTypeDiff, this.decorations)

client/cody/test/e2e/BUILD.bazel

@@ -0,0 +1,23 @@
+# Cody E2E Testing
+
+See the [client/cody/CONTRIBUTING.md](../../CONTRIBUTING.md) for more information.
+
+### Commands
+
+#### All Tests
+
+```sh
+pnpm test:e2e
+```
+
+#### Individual Test
+
+```sh
+pnpm test:e2e $TEST_NAME
+```
+
+#### Debug Test
+
+```sh
+pnpm test:e2e --debug
+```

client/cody/test/e2e/README.md

@@ -0,0 +1,16 @@
+import { Frame, Locator, Page, expect } from '@playwright/test'
+
+import { SERVER_URL, VALID_TOKEN } from '../fixtures/mock-server'
+
+// Sign into Cody with valid auth from the sidebar
+export const sidebarSignin = async (sidebar: Frame): Promise<void> => {
+    await sidebar.getByRole('textbox', { name: 'Sourcegraph Instance URL' }).fill(SERVER_URL)
+    await sidebar.getByRole('textbox', { name: 'Access Token (docs)' }).fill(VALID_TOKEN)
+    await sidebar.getByRole('button', { name: 'Sign In' }).click()
+
+    await expect(sidebar.getByText("Hello! I'm Cody.")).toBeVisible()
+}
+
+// Selector for the Explorer button in the sidebar that would match on Mac and Linux
+const sidbarExplorerRole = { name: /Explorer.*/ }
+export const sidebarExplorer = (page: Page): Locator => page.getByRole('tab', sidbarExplorerRole)

client/cody/test/e2e/common.ts

@@ -1,4 +1,4 @@
-import { mkdtempSync, rmdirSync } from 'fs'
+import { mkdir, mkdtempSync, rmdirSync, writeFile } from 'fs'
 import { tmpdir } from 'os'
 import * as path from 'path'
 
@@ -24,6 +24,8 @@ export const test = base
 
             const workspaceDirectory = path.join(codyRoot, 'test', 'fixtures', 'workspace')
 
+            await buildWorkSpaceSettings(workspaceDirectory)
+
             // See: https://github.com/microsoft/vscode-test/blob/main/lib/runTest.ts
             const app = await electron.launch({
                 executablePath: vscodeExecutablePath,
@@ -113,7 +115,7 @@ export async function getCodySidebar(page: Page): Promise<Frame> {
         return null
     }
     await waitUntil(async () => (await findCodySidebarFrame()) !== null)
-    return (await findCodySidebarFrame())!
+    return (await findCodySidebarFrame()) || page.mainFrame()
 }
 
 async function waitUntil(predicate: () => boolean | Promise<boolean>): Promise<void> {
@@ -127,3 +129,23 @@ async function waitUntil(predicate: () => boolean | Promise<boolean>): Promise<v
 function escapeToPath(text: string): string {
     return text.replace(/\W/g, '_')
 }
+
+// Build a workspace settings file that enables the experimental inline mode
+export async function buildWorkSpaceSettings(workspaceDirectory: string): Promise<void> {
+    const settings = {
+        'cody.experimental.inline': true,
+    }
+    // create a temporary directory with settings.json and add to the workspaceDirectory
+    const workspaceSettingsPath = path.join(workspaceDirectory, '.vscode', 'settings.json')
+    const workspaceSettingsDirectory = path.join(workspaceDirectory, '.vscode')
+    mkdir(workspaceSettingsDirectory, { recursive: true }, () => {})
+    await new Promise<void>((resolve, reject) => {
+        writeFile(workspaceSettingsPath, JSON.stringify(settings), error => {
+            if (error) {
+                reject(error)
+            } else {
+                resolve()
+            }
+        })
+    })
+}

client/cody/test/e2e/helpers.ts

@@ -0,0 +1,44 @@
+import { expect } from '@playwright/test'
+
+import { sidebarExplorer, sidebarSignin } from './common'
+import { test } from './helpers'
+
+test('start a fixup job from inline assist with valid auth', async ({ page, sidebar }) => {
+    // Sign into Cody
+    await sidebarSignin(sidebar)
+
+    // Open the Explorer view from the sidebar
+    await sidebarExplorer(page).click()
+
+    // Select the second files from the tree view, which is the index.html file
+    await page.locator('.monaco-highlighted-label').nth(2).click()
+
+    // Click on line number 6 to open the comment thread
+    await page.locator('.comment-diff-added').nth(5).hover()
+    await page.locator('.comment-diff-added').nth(5).click()
+
+    // After opening the comment thread, we need to wait for the editor to load
+    await page.waitForSelector('.monaco-editor')
+    await page.waitForSelector('.monaco-text-button')
+
+    // Type in the instruction for fixup
+    await page.keyboard.type('/fix replace hello with goodbye')
+    // Click on the submit button with the name Ask Cody
+    await page.click('.monaco-text-button')
+
+    // TODO: Capture processing state. It is currently to quick to capture the processing elements
+    // Wait for the code lens to show up to ensure that the fixup has been applied
+    // await expect(page.getByText('Processing by Cody')).toBeVisible()
+
+    // Ensures Code Lens is added
+    await expect(page.getByText('Edited by Cody')).toBeVisible()
+    await expect(page.getByText('<title>Goodbye Cody</title>')).toBeVisible()
+
+    // Ensures Cody's fixup is displayed in comment thread
+    await expect(page.getByText('Check your document for updates from Cody.')).toBeVisible()
+
+    // Ensures Decorations is displayed by checking hover text
+    await page.getByText('>Goodbye Cody<').hover()
+    // The decoration text on hover should start with 'Cody Fixup #' and end with random number
+    await page.getByRole('tooltip', { name: /Cody Assist.*/ }).click()
+})