e2e-test: browser in parallel & enable Currents

.github/workflows/e2e-test-release-run-ubuntu.yml

@@ -88,10 +88,15 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
-        id: electron-smoke-tests
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron,release,${{ inputs.e2e_grep }}"
+        id: electron-e2e-tests
         run: |
           export DISPLAY=:10
-          BUILD=/usr/share/positron npx playwright test --project e2e-electron --workers 2 --grep="${{ env.E2E_GREP }}"
+          BUILD=/usr/share/positron npx playwright test --project e2e-electron --workers 3 --grep=${{ env.E2E_GREP }}
 
       - name: Upload Playwright Report to S3
         if: ${{ !cancelled() }}

.github/workflows/positron-full-test.yml

@@ -96,11 +96,6 @@ jobs:
   e2e-electron-tests:
     runs-on: ubuntu-latest-8x
     timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        shardIndex: [1, 2]
-        shardTotal: [2]
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       POSITRON_BUILD_NUMBER: 0 # CI skips building releases
@@ -131,26 +126,32 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_PROJECT_ID: ZOs5z2
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }} # only works on push events
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron"
         id: electron-tests
-        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 1 --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 3
 
       - name: Upload blob report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: blob-report-electron-${{ matrix.shardIndex }}
+          name: blob-report-electron
           path: blob-report
           retention-days: 14
 
       - name: Upload junit report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: junit-report-electron-${{ matrix.shardIndex }}
+          name: junit-report-electron
           path: test-results/junit.xml
 
   e2e-browser-tests:
-    runs-on: ubuntu-latest-4x
+    runs-on: ubuntu-latest-8x
     timeout-minutes: 50
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -188,8 +189,14 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_PROJECT_ID: ZOs5z2
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "chromium"
         id: browser-tests
-        run: DISPLAY=:10 npx playwright test --project e2e-browser --workers 1
+        run: DISPLAY=:10 npx playwright test --project e2e-browser --workers 2
 
       - name: Upload blob report
         if: ${{ !cancelled() }}

.github/workflows/positron-merge-to-branch.yml

@@ -70,8 +70,13 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron,${{ inputs.e2e_grep }}"
         id: e2e-playwright-tests
-        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 2 --grep="${{ env.E2E_GREP }}"
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 2 --grep=${{ env.E2E_GREP }}
 
       - name: Upload Playwright Report to S3
         if: ${{ success() || failure() }}

.github/workflows/positron-windows-nightly.yml

@@ -102,23 +102,29 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.10
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_PROJECT_ID: ZOs5z2
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }} # only works on push events
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron,@win"
         if: ${{ !cancelled() }}
         id: e2e-win-electron-tests
-        run: npx playwright test --project e2e-electron --grep "@win" --workers 1
+        run: npx playwright test --project e2e-electron --grep "@win" --workers 2
 
       - name: Upload blob report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: blob-report-electron-${{ matrix.shardIndex }}
+          name: blob-report-electron
           path: blob-report
           retention-days: 14
 
       - name: Upload junit report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: junit-report-electron-${{ matrix.shardIndex }}
+          name: junit-report-electron
           path: test-results/junit.xml
 
   e2e-report:

package.json

@@ -46,7 +46,7 @@
     "update-localization-extension": "node build/npm/update-localization-extension.js",
     "e2e": "yarn e2e-electron",
     "e2e-electron": "npx playwright test --project e2e-electron",
-    "e2e-browser": "npx playwright test --project e2e-browser --workers 1",
+    "e2e-browser": "npx playwright test --project e2e-browser",
     "e2e-pr": "npx playwright test --project e2e-electron --grep @pr",
     "e2e-win": "npx playwright test --project e2e-electron --grep @win",
     "e2e-failed": "npx playwright test --last-failed",
@@ -129,6 +129,7 @@
     "yazl": "^2.4.3"
   },
   "devDependencies": {
+    "@currents/playwright": "^1.8.0",
     "@midleman/github-actions-reporter": "^1.9.5",
     "@playwright/test": "^1.49.0",
     "@swc/core": "1.3.62",

playwright.config.ts

@@ -6,6 +6,7 @@
 import { defineConfig } from '@playwright/test';
 import { CustomTestOptions } from './test/smoke/src/areas/positron/_test.setup';
 import type { GitHubActionOptions } from '@midleman/github-actions-reporter';
+import { currentsReporter } from '@currents/playwright';
 
 /**
  * See https://playwright.dev/docs/test-configuration.
@@ -37,7 +38,13 @@ export default defineConfig<CustomTestOptions>({
 				includeResults: ['fail', 'flaky']
 			}],
 			['junit', { outputFile: 'test-results/junit.xml' }],
-			['list'], ['html'], ['blob']
+			['list'], ['html'], ['blob'],
+			currentsReporter({
+				ciBuildId: process.env.CURRENTS_CI_BUILD_ID || Date.now().toString(),
+				recordKey: process.env.CURRENTS_RECORD_KEY || '',
+				projectId: 'ZOs5z2',
+				disableTitleTags: true,
+			}),
 		]
 		: [
 			['list'],

test/automation/src/playwrightBrowser.ts

@@ -30,66 +30,115 @@ export async function launch(options: LaunchOptions): Promise<{ serverProcess: C
 	};
 }
 
+// --- Start Positron ---
+// Modified `launchServer` function to add support for multiple ports to enable parallel test
+// execution of browser tests. Also added helper functions: `getServerArgs`, `resolveServerLocation`,
+// and `startServer` to make this code easier to read.
 async function launchServer(options: LaunchOptions) {
 	const { userDataDir, codePath, extensionsPath, logger, logsPath } = options;
 	const serverLogsPath = join(logsPath, 'server');
 	const codeServerPath = codePath ?? process.env.VSCODE_REMOTE_SERVER_PATH;
 	const agentFolder = userDataDir;
+
 	await measureAndLog(() => mkdirp(agentFolder), `mkdirp(${agentFolder})`, logger);
 
 	const env = {
 		VSCODE_REMOTE_SERVER_PATH: codeServerPath,
-		...process.env
+		...process.env,
 	};
 
+	const maxRetries = 10;
+	let serverProcess: ChildProcess | null = null;
+	let endpoint: string | undefined;
+
+	for (let attempts = 0; attempts < maxRetries; attempts++) {
+		const currentPort = port++;
+		const args = getServerArgs(currentPort, extensionsPath, agentFolder, serverLogsPath, options.verbose);
+		const serverLocation = resolveServerLocation(codeServerPath, logger);
+
+		logger.log(`Attempting to start server on port ${currentPort}`);
+		logger.log(`Command: '${serverLocation}' ${args.join(' ')}`);
+
+		try {
+			serverProcess = await startServer(serverLocation, args, env, logger);
+			endpoint = await measureAndLog(
+				() => waitForEndpoint(serverProcess!, logger),
+				'waitForEndpoint(serverProcess)',
+				logger
+			);
+
+			logger.log(`Server started successfully on port ${currentPort}`);
+			break; // Exit loop on success
+		} catch (error) {
+			if ((error as Error).message.includes('EADDRINUSE')) {
+				logger.log(`Port ${currentPort} is already in use. Retrying...`);
+				serverProcess?.kill();
+			} else {
+				throw error; // Rethrow non-port-related errors
+			}
+		}
+	}
+
+	if (!serverProcess || !endpoint) {
+		throw new Error('Failed to launch the server after multiple attempts.');
+	}
+
+	return { serverProcess, endpoint };
+}
+
+function getServerArgs(
+	port: number,
+	extensionsPath: string,
+	agentFolder: string,
+	logsPath: string,
+	verbose?: boolean
+): string[] {
 	const args = [
 		'--disable-telemetry',
 		'--disable-workspace-trust',
-		`--port=${port++}`,
+		`--port=${port}`,
 		'--enable-smoke-test-driver',
 		`--extensions-dir=${extensionsPath}`,
 		`--server-data-dir=${agentFolder}`,
 		'--accept-server-license-terms',
-		`--logsPath=${serverLogsPath}`,
-		// --- Start Positron ---
-		`--connection-token`,
-		`dev-token`
-		// --- End Positron ---
+		`--logsPath=${logsPath}`,
+		'--connection-token',
+		'dev-token',
 	];
 
-	if (options.verbose) {
+	if (verbose) {
 		args.push('--log=trace');
 	}
 
-	let serverLocation: string | undefined;
+	return args;
+}
+
+function resolveServerLocation(codeServerPath: string | undefined, logger: Logger): string {
 	if (codeServerPath) {
 		const { serverApplicationName } = require(join(codeServerPath, 'product.json'));
-		serverLocation = join(codeServerPath, 'bin', `${serverApplicationName}${process.platform === 'win32' ? '.cmd' : ''}`);
-
-		logger.log(`Starting built server from '${serverLocation}'`);
-	} else {
-		serverLocation = join(root, `scripts/code-server.${process.platform === 'win32' ? 'bat' : 'sh'}`);
-
-		logger.log(`Starting server out of sources from '${serverLocation}'`);
+		const serverLocation = join(codeServerPath, 'bin', `${serverApplicationName}${process.platform === 'win32' ? '.cmd' : ''}`);
+		logger.log(`Using built server from '${serverLocation}'`);
+		return serverLocation;
 	}
 
-	logger.log(`Storing log files into '${serverLogsPath}'`);
-
-	logger.log(`Command line: '${serverLocation}' ${args.join(' ')}`);
-	const shell: boolean = (process.platform === 'win32');
-	const serverProcess = spawn(
-		serverLocation,
-		args,
-		{ env, shell }
-	);
-
-	logger.log(`Started server for browser smoke tests (pid: ${serverProcess.pid})`);
+	const scriptPath = join(root, `scripts/code-server.${process.platform === 'win32' ? 'bat' : 'sh'}`);
+	logger.log(`Using source server from '${scriptPath}'`);
+	return scriptPath;
+}
 
-	return {
-		serverProcess,
-		endpoint: await measureAndLog(() => waitForEndpoint(serverProcess, logger), 'waitForEndpoint(serverProcess)', logger)
-	};
+async function startServer(
+	serverLocation: string,
+	args: string[],
+	env: NodeJS.ProcessEnv,
+	logger: Logger
+): Promise<ChildProcess> {
+	logger.log(`Starting server: ${serverLocation}`);
+	const serverProcess = spawn(serverLocation, args, { env, shell: process.platform === 'win32' });
+	logger.log(`Server started (pid: ${serverProcess.pid})`);
+	return serverProcess;
 }
+// --- End Positron ---
+
 
 async function launchBrowser(options: LaunchOptions, endpoint: string) {
 	const { logger, workspacePath, tracing, headless } = options;

test/automation/src/positron/positronNotebooks.ts

@@ -82,7 +82,7 @@ export class PositronNotebooks {
 	}
 
 	async assertCellOutput(text: string): Promise<void> {
-		await expect(this.frameLocator.getByText(text)).toBeVisible();
+		await expect(this.frameLocator.getByText(text)).toBeVisible({ timeout: 15000 });
 	}
 
 	async closeNotebookWithoutSaving() {

test/automation/src/positron/positronVariables.ts

@@ -57,7 +57,6 @@ export class PositronVariables {
 	}
 
 	async doubleClickVariableRow(variableName: string) {
-
 		const desiredRow = await this.waitForVariableRow(variableName);
 		await desiredRow.dblclick();
 	}