Merge pull request #3328 from continuedev/tomasz/e2e-completions

E2E Completion Test

extensions/vscode/e2e/TestUtils.ts

@@ -2,11 +2,11 @@ import { expect } from "chai";
 import { WebElement } from "vscode-extension-tester";
 
 export class TestUtils {
-  public static async waitForElement(
-    locatorFn: () => Promise<WebElement>,
+  public static async waitForElement<T>(
+    locatorFn: () => Promise<T>,
     timeout: number = 5000,
     interval: number = 500,
-  ): Promise<WebElement> {
+  ): Promise<T> {
     const startTime = Date.now();
 
     while (Date.now() - startTime < timeout) {

extensions/vscode/e2e/actions/Global.actions.ts

@@ -0,0 +1,8 @@
+import { VSBrowser, WebView } from "vscode-extension-tester";
+import * as path from "path";
+
+export class GlobalActions {
+  public static async openTestWorkspace() {
+    return VSBrowser.instance.openResources(path.join("e2e/test-continue"));
+  }
+}

extensions/vscode/e2e/constants.ts

@@ -0,0 +1 @@
+export const DEFAULT_TIMEOUT = 30_000;

extensions/vscode/e2e/selectors/Autocomplete.selectors.ts

@@ -0,0 +1,11 @@
+import { By, WebDriver } from "vscode-extension-tester";
+
+export class AutocompleteSelectors {
+  public static async getGhostTextContent(driver: WebDriver) {
+    const element = await driver.findElement(
+      By.xpath("//span[contains(@class, 'ghost-text-decoration')]"),
+    );
+
+    return element.getText();
+  }
+}

extensions/vscode/e2e/test-continue/config.json

@@ -18,10 +18,9 @@
     "provider": "continue-proxy"
   },
   "tabAutocompleteModel": {
-    "title": "Qwen",
-    "provider": "mistral",
-    "model": "codestral-latest",
-    "apiKey": ""
+    "title": "TEST LLM",
+    "provider": "test",
+    "model": "this field is not used"
   },
   "tabAutocompleteOptions": {
     "useCache": false

extensions/vscode/e2e/tests/AutocompleteFinal.test.ts

@@ -0,0 +1,55 @@
+import {
+  EditorView,
+  TextEditor,
+  InputBox,
+  Workbench,
+} from "vscode-extension-tester";
+
+import { TestUtils } from "../TestUtils";
+import { DEFAULT_TIMEOUT } from "../constants";
+import { AutocompleteSelectors } from "../selectors/Autocomplete.selectors";
+import { expect } from "chai";
+import { GlobalActions } from "../actions/Global.actions";
+
+describe("Autocomplete", () => {
+  let editor: TextEditor;
+
+  beforeEach(async function () {
+    this.timeout(DEFAULT_TIMEOUT);
+
+    await GlobalActions.openTestWorkspace();
+    await new Workbench().executeCommand("Create: New File...");
+    await (await InputBox.create()).selectQuickPick("Text File");
+    editor = (await new EditorView().openEditor("Untitled-1")) as TextEditor;
+  });
+
+  afterEach(async function () {
+    this.timeout(DEFAULT_TIMEOUT);
+    await editor.clearText();
+    await new EditorView().closeAllEditors();
+  });
+
+  it("Should display completions", async () => {
+    const driver = editor.getDriver();
+
+    const messagePair0 = TestUtils.generateTestMessagePair(0);
+    await editor.typeTextAt(1, 1, messagePair0.userMessage);
+    await editor.typeTextAt(1, messagePair0.userMessage.length + 1, " ");
+    const ghostText0 = await TestUtils.waitForElement(
+      () => AutocompleteSelectors.getGhostTextContent(driver),
+      // The first completion takes longer because Continue needs to load
+      DEFAULT_TIMEOUT,
+    );
+    expect(ghostText0).to.equal(messagePair0.llmResponse);
+
+    await editor.clearText();
+
+    const messagePair1 = TestUtils.generateTestMessagePair(1);
+    await editor.typeTextAt(1, 1, messagePair1.userMessage);
+    await editor.typeTextAt(1, messagePair1.userMessage.length + 1, " ");
+    const ghostText1 = await TestUtils.waitForElement(() =>
+      AutocompleteSelectors.getGhostTextContent(driver),
+    );
+    expect(ghostText1).to.equal(messagePair1.llmResponse);
+  }).timeout(DEFAULT_TIMEOUT);
+});

extensions/vscode/e2e/tests/GUI.test.ts

@@ -1,25 +1,18 @@
-import {
-  EditorView,
-  WebView,
-  WebDriver,
-  VSBrowser,
-  Key,
-} from "vscode-extension-tester";
+import { EditorView, WebView, WebDriver, Key } from "vscode-extension-tester";
 import { expect } from "chai";
 import { GUIActions } from "../actions/GUI.actions";
 import { GUISelectors } from "../selectors/GUI.selectors";
-import * as path from "path";
 import { TestUtils } from "../TestUtils";
-
-const DEFAULT_TIMEOUT = 30_000;
+import { DEFAULT_TIMEOUT } from "../constants";
+import { GlobalActions } from "../actions/Global.actions";
 
 describe("GUI Test", () => {
   let view: WebView;
   let driver: WebDriver;
 
   before(async function () {
     this.timeout(DEFAULT_TIMEOUT);
-    await VSBrowser.instance.openResources(path.join("e2e/test-continue"));
+    await GlobalActions.openTestWorkspace();
   });
 
   beforeEach(async function () {
@@ -76,7 +69,7 @@ describe("GUI Test", () => {
       );
     });
 
-    it.only("Can delete messages", async () => {
+    it("Can delete messages", async () => {
       const { userMessage: userMessage0, llmResponse: llmResponse0 } =
         TestUtils.generateTestMessagePair(0);
       await GUIActions.sendMessage({