[C3] Address e2e flakiness

.changeset/strong-ways-destroy.md

@@ -0,0 +1,5 @@
+---
+"create-cloudflare": minor
+---
+
+Verifies the existence of pages projects before attempting to deploy for added reliability.

.github/workflows/test-c3.yml

@@ -6,7 +6,7 @@ on:
       - packages/create-cloudflare/**
 
 env:
-  node-version: 16.14
+  node-version: 18.17.1
 
 jobs:
   check:
@@ -36,6 +36,7 @@ jobs:
         run: npm run test:unit -w create-cloudflare
 
   cleanup:
+    name: "Cleanup Test Projects"
     if: ${{ github.repository_owner == 'cloudflare' }}
     runs-on: ubuntu-latest
     steps:
@@ -55,16 +56,15 @@ jobs:
 
   e2e:
     needs: cleanup
-    name: "E2E"
+    name: ${{ format('E2E ({0}) {1}', matrix.pm, matrix.quarantine == true && '[quarantine]' || '') }}
+    continue-on-error: ${{ matrix.quarantine }}
     if: |
       github.event.pull_request.user.login != 'dependabot[bot]'
     strategy:
       matrix:
-        # TODO: add back windows
-        # os: [ubuntu-latest, windows-latest, macos-latest]
         os: [ubuntu-latest]
-        # pm: [npm, yarn, pnpm]
         pm: [npm, pnpm]
+        quarantine: [true, false]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout Repo
@@ -80,8 +80,10 @@ jobs:
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}
+          E2E_QUARANTINE: ${{ matrix.quarantine }}
 
   get-dependabot-bumped-framework:
+    name: "Get bumped framework (dependabot-only)"
     runs-on: ubuntu-latest
     outputs:
       bumped-framework-cli: ${{ steps.detect.outputs.result }}
@@ -129,14 +131,13 @@ jobs:
   # framework (this is both for optimization and in order to reduce unnecessary flakiness)
   e2e-only-dependabot-bumped-framework:
     needs: [cleanup, get-dependabot-bumped-framework]
-    name: "Dependabot specific E2E"
+    name: ${{ format('Dependabot E2E ({0}) {1}', matrix.pm, matrix.quarantine == true && '[quarantine]' || '') }}
+    continue-on-error: ${{ matrix.quarantine }}
     strategy:
       matrix:
-        # TODO: add back windows
-        # os: [ubuntu-latest, windows-latest, macos-latest]
         os: [ubuntu-latest]
-        # pm: [npm, yarn, pnpm]
         pm: [npm, pnpm]
+        quarantine: [true, false]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout Repo
@@ -153,3 +154,4 @@ jobs:
           FRAMEWORK_CLI_TO_TEST: ${{ needs.get-bumped-framework.outputs.bumped-framework-cli }}
           CLOUDFLARE_API_TOKEN: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}
+          E2E_QUARANTINE: ${{ matrix.quarantine }}

packages/create-cloudflare/e2e-tests/cli.test.ts

@@ -4,124 +4,128 @@ import { join } from "path";
 import { beforeEach, afterEach, describe, test, expect } from "vitest";
 import { version } from "../package.json";
 import { frameworkToTest } from "./frameworkToTest";
-import { keys, runC3 } from "./helpers";
+import { isQuarantineMode, keys, runC3 } from "./helpers";
 
 // Note: skipIf(frameworkToTest) makes it so that all the basic C3 functionality
 //       tests are skipped in case we are testing a specific framework
-describe.skipIf(frameworkToTest)("E2E: Basic C3 functionality ", () => {
-	const tmpDirPath = realpathSync(mkdtempSync(join(tmpdir(), "c3-tests")));
-	const projectPath = join(tmpDirPath, "basic-tests");
+describe.skipIf(frameworkToTest || isQuarantineMode())(
+	"E2E: Basic C3 functionality ",
+	() => {
+		const tmpDirPath = realpathSync(mkdtempSync(join(tmpdir(), "c3-tests")));
+		const projectPath = join(tmpDirPath, "basic-tests");
 
-	beforeEach(() => {
-		rmSync(projectPath, { recursive: true, force: true });
-	});
-
-	afterEach(() => {
-		if (existsSync(projectPath)) {
-			rmSync(projectPath, { recursive: true });
-		}
-	});
+		beforeEach(() => {
+			rmSync(projectPath, { recursive: true, force: true });
+		});
 
-	test("--version", async () => {
-		const { output } = await runC3({ argv: ["--version"] });
-		expect(output).toEqual(version);
-	});
+		afterEach(() => {
+			if (existsSync(projectPath)) {
+				rmSync(projectPath, { recursive: true });
+			}
+		});
 
-	test("--version with positionals", async () => {
-		const argv = "foo bar baz --version".split(" ");
-		const { output } = await runC3({ argv });
-		expect(output).toEqual(version);
-	});
+		test("--version", async () => {
+			const { output } = await runC3({ argv: ["--version"] });
+			expect(output).toEqual(version);
+		});
 
-	test("--version with flags", async () => {
-		const argv = "foo --type webFramework --no-deploy --version".split(" ");
-		const { output } = await runC3({ argv });
-		expect(output).toEqual(version);
-	});
+		test("--version with positionals", async () => {
+			const argv = "foo bar baz --version".split(" ");
+			const { output } = await runC3({ argv });
+			expect(output).toEqual(version);
+		});
 
-	test("Using arrow keys + enter", async () => {
-		const { output } = await runC3({
-			argv: [projectPath],
-			promptHandlers: [
-				{
-					matcher: /What type of application do you want to create/,
-					input: [keys.enter],
-				},
-				{
-					matcher: /Do you want to use TypeScript/,
-					input: [keys.enter],
-				},
-				{
-					matcher: /Do you want to use git for version control/,
-					input: [keys.right, keys.enter],
-				},
-				{
-					matcher: /Do you want to deploy your application/,
-					input: [keys.left, keys.enter],
-				},
-			],
+		test("--version with flags", async () => {
+			const argv = "foo --type webFramework --no-deploy --version".split(" ");
+			const { output } = await runC3({ argv });
+			expect(output).toEqual(version);
 		});
 
-		expect(projectPath).toExist();
-		expect(output).toContain(`type "Hello World" Worker`);
-		expect(output).toContain(`yes typescript`);
-		expect(output).toContain(`no git`);
-		expect(output).toContain(`no deploy`);
-	});
+		test("Using arrow keys + enter", async () => {
+			const { output } = await runC3({
+				argv: [projectPath],
+				promptHandlers: [
+					{
+						matcher: /What type of application do you want to create/,
+						input: [keys.enter],
+					},
+					{
+						matcher: /Do you want to use TypeScript/,
+						input: [keys.enter],
+					},
+					{
+						matcher: /Do you want to use git for version control/,
+						input: [keys.right, keys.enter],
+					},
+					{
+						matcher: /Do you want to deploy your application/,
+						input: [keys.left, keys.enter],
+					},
+				],
+			});
 
-	test("Typing custom responses", async () => {
-		const { output } = await runC3({
-			argv: [],
-			promptHandlers: [
-				{
-					matcher: /In which directory do you want to create your application/,
-					input: [projectPath, keys.enter],
-				},
-				{
-					matcher: /What type of application do you want to create/,
-					input: [keys.down, keys.enter],
-				},
-				{
-					matcher: /Do you want to use TypeScript/,
-					input: ["n"],
-				},
-				{
-					matcher: /Do you want to use git for version control/,
-					input: ["n"],
-				},
-				{
-					matcher: /Do you want to deploy your application/,
-					input: ["n"],
-				},
-			],
+			expect(projectPath).toExist();
+			expect(output).toContain(`type "Hello World" Worker`);
+			expect(output).toContain(`yes typescript`);
+			expect(output).toContain(`no git`);
+			expect(output).toContain(`no deploy`);
 		});
 
-		expect(projectPath).toExist();
-		expect(output).toContain(`type Example router & proxy Worker`);
-		expect(output).toContain(`no typescript`);
-		expect(output).toContain(`no git`);
-		expect(output).toContain(`no deploy`);
-	});
+		test("Typing custom responses", async () => {
+			const { output } = await runC3({
+				argv: [],
+				promptHandlers: [
+					{
+						matcher:
+							/In which directory do you want to create your application/,
+						input: [projectPath, keys.enter],
+					},
+					{
+						matcher: /What type of application do you want to create/,
+						input: [keys.down, keys.enter],
+					},
+					{
+						matcher: /Do you want to use TypeScript/,
+						input: ["n"],
+					},
+					{
+						matcher: /Do you want to use git for version control/,
+						input: ["n"],
+					},
+					{
+						matcher: /Do you want to deploy your application/,
+						input: ["n"],
+					},
+				],
+			});
 
-	test("Mixed args and interactive", async () => {
-		const { output } = await runC3({
-			argv: [projectPath, "--ts", "--no-deploy"],
-			promptHandlers: [
-				{
-					matcher: /What type of application do you want to create/,
-					input: [keys.enter],
-				},
-				{
-					matcher: /Do you want to use git for version control/,
-					input: ["n"],
-				},
-			],
+			expect(projectPath).toExist();
+			expect(output).toContain(`type Example router & proxy Worker`);
+			expect(output).toContain(`no typescript`);
+			expect(output).toContain(`no git`);
+			expect(output).toContain(`no deploy`);
 		});
 
-		expect(projectPath).toExist();
-		expect(output).toContain(`type "Hello World" Worker`);
-		expect(output).toContain(`yes typescript`);
-		expect(output).toContain(`no git`);
-		expect(output).toContain(`no deploy`);
-	});
-});
+		test("Mixed args and interactive", async () => {
+			const { output } = await runC3({
+				argv: [projectPath, "--ts", "--no-deploy"],
+				promptHandlers: [
+					{
+						matcher: /What type of application do you want to create/,
+						input: [keys.enter],
+					},
+					{
+						matcher: /Do you want to use git for version control/,
+						input: ["n"],
+					},
+				],
+			});
+
+			expect(projectPath).toExist();
+			expect(output).toContain(`type "Hello World" Worker`);
+			expect(output).toContain(`yes typescript`);
+			expect(output).toContain(`no git`);
+			expect(output).toContain(`no deploy`);
+		});
+	}
+);

packages/create-cloudflare/e2e-tests/helpers.ts

@@ -3,6 +3,7 @@ import crypto from "node:crypto";
 import { tmpdir } from "os";
 import { join } from "path";
 import { spawn } from "cross-spawn";
+import { sleep } from "helpers/common";
 import { spinnerFrames } from "helpers/interactive";
 import type { SpinnerStyle } from "helpers/interactive";
 
@@ -30,6 +31,7 @@ export type RunnerConfig = {
 	promptHandlers?: PromptHandler[];
 	argv?: string[];
 	outputPrefix?: string;
+	quarantine?: boolean;
 };
 
 export const runC3 = async ({
@@ -46,14 +48,17 @@ export const runC3 = async ({
 			const lines: string[] = data.toString().split("\n");
 			const currentDialog = promptHandlers[0];
 
-			lines.forEach((line) => {
+			lines.forEach(async (line) => {
 				// Uncomment to debug test output
 				if (filterLine(line)) {
 					console.log(`${outputPrefix} ${line}`);
 				}
 				stdout.push(line);
 
 				if (currentDialog && currentDialog.matcher.test(line)) {
+					// Add a small sleep to avoid input race
+					await sleep(500);
+
 					currentDialog.input.forEach((keystroke) => {
 						proc.stdin.write(keystroke);
 					});
@@ -103,7 +108,7 @@ export const testProjectDir = (suite: string) => {
 		mkdtempSync(join(tmpdir(), `c3-tests-${suite}`))
 	);
 
-	const randomSuffix = crypto.randomBytes(3).toString("hex");
+	const randomSuffix = crypto.randomBytes(4).toString("hex");
 	const baseProjectName = `${C3_E2E_PREFIX}${randomSuffix}`;
 
 	const getName = (suffix: string) => `${baseProjectName}-${suffix}`;
@@ -136,3 +141,7 @@ const filterLine = (line: string) => {
 
 	return true;
 };
+
+export const isQuarantineMode = () => {
+	return process.env.E2E_QUARANTINE === "true";
+};