Skip to content

Commit

Permalink
[C3] Address e2e flakiness (#3877)
Browse files Browse the repository at this point in the history 
* C3: Fix input hang in e2e tests

* C3: Verify project creation before deploying

* C3: Bump node version in e2e tests

* fix lint issue

* bump astro version

* increase concurrency level from 3 to 4

* Adjust concurrency level back to 3

* Add support for e2e test quarantine

* Make quarantine runs a separate job

* Add a changeset

* Adding more logging for project validation

* Quarantining astro. Adjusing changeset

* refactor quarantine workflow

* Refactor dependabot e2e quarantine workflow

* Fix formatting issue in e2e job name

* Second attempt to fix name formatting issue in e2e workflow

* remove changeset

* Remove astro from quarantine

* Fixing quarantine mode check
  • Loading branch information
@jculvey
jculvey authored Sep 5, 2023
1 parent ae2d5cb commit 975cbd9
Show file tree
Hide file tree
Showing 11 changed files with 255 additions and 176 deletions.
20 changes: 11 additions & 9 deletions .github/workflows/test-c3.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ on:
- packages/create-cloudflare/**

env:
node-version: 16.14
node-version: 18.17.1

jobs:
check:
Expand Down Expand Up @@ -36,6 +36,7 @@ jobs:
run: npm run test:unit -w create-cloudflare

cleanup:
name: "Cleanup Test Projects"
if: ${{ github.repository_owner == 'cloudflare' }}
runs-on: ubuntu-latest
steps:
Expand All @@ -55,16 +56,15 @@ jobs:

e2e:
needs: cleanup
name: "E2E"
name: ${{ format('E2E ({0}) {1}', matrix.pm, matrix.quarantine == true && '[quarantine]' || '') }}
continue-on-error: ${{ matrix.quarantine }}
if: |
github.event.pull_request.user.login != 'dependabot[bot]'
strategy:
matrix:
# TODO: add back windows
# os: [ubuntu-latest, windows-latest, macos-latest]
os: [ubuntu-latest]
# pm: [npm, yarn, pnpm]
pm: [npm, pnpm]
quarantine: [true, false]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout Repo
Expand All @@ -80,8 +80,10 @@ jobs:
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}
E2E_QUARANTINE: ${{ matrix.quarantine }}

get-dependabot-bumped-framework:
name: "Get bumped framework (dependabot-only)"
runs-on: ubuntu-latest
outputs:
bumped-framework-cli: ${{ steps.detect.outputs.result }}
Expand Down Expand Up @@ -129,14 +131,13 @@ jobs:
# framework (this is both for optimization and in order to reduce unnecessary flakiness)
e2e-only-dependabot-bumped-framework:
needs: [cleanup, get-dependabot-bumped-framework]
name: "Dependabot specific E2E"
name: ${{ format('Dependabot E2E ({0}) {1}', matrix.pm, matrix.quarantine == true && '[quarantine]' || '') }}
continue-on-error: ${{ matrix.quarantine }}
strategy:
matrix:
# TODO: add back windows
# os: [ubuntu-latest, windows-latest, macos-latest]
os: [ubuntu-latest]
# pm: [npm, yarn, pnpm]
pm: [npm, pnpm]
quarantine: [true, false]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout Repo
Expand All @@ -153,3 +154,4 @@ jobs:
FRAMEWORK_CLI_TO_TEST: ${{ needs.get-bumped-framework.outputs.bumped-framework-cli }}
CLOUDFLARE_API_TOKEN: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}
E2E_QUARANTINE: ${{ matrix.quarantine }}
214 changes: 109 additions & 105 deletions packages/create-cloudflare/e2e-tests/cli.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,124 +4,128 @@ import { join } from "path";
import { beforeEach, afterEach, describe, test, expect } from "vitest";
import { version } from "../package.json";
import { frameworkToTest } from "./frameworkToTest";
import { keys, runC3 } from "./helpers";
import { isQuarantineMode, keys, runC3 } from "./helpers";

// Note: skipIf(frameworkToTest) makes it so that all the basic C3 functionality
// tests are skipped in case we are testing a specific framework
describe.skipIf(frameworkToTest)("E2E: Basic C3 functionality ", () => {
const tmpDirPath = realpathSync(mkdtempSync(join(tmpdir(), "c3-tests")));
const projectPath = join(tmpDirPath, "basic-tests");
describe.skipIf(frameworkToTest || isQuarantineMode())(
"E2E: Basic C3 functionality ",
() => {
const tmpDirPath = realpathSync(mkdtempSync(join(tmpdir(), "c3-tests")));
const projectPath = join(tmpDirPath, "basic-tests");

beforeEach(() => {
rmSync(projectPath, { recursive: true, force: true });
});

afterEach(() => {
if (existsSync(projectPath)) {
rmSync(projectPath, { recursive: true });
}
});
beforeEach(() => {
rmSync(projectPath, { recursive: true, force: true });
});

test("--version", async () => {
const { output } = await runC3({ argv: ["--version"] });
expect(output).toEqual(version);
});
afterEach(() => {
if (existsSync(projectPath)) {
rmSync(projectPath, { recursive: true });
}
});

test("--version with positionals", async () => {
const argv = "foo bar baz --version".split(" ");
const { output } = await runC3({ argv });
expect(output).toEqual(version);
});
test("--version", async () => {
const { output } = await runC3({ argv: ["--version"] });
expect(output).toEqual(version);
});

test("--version with flags", async () => {
const argv = "foo --type webFramework --no-deploy --version".split(" ");
const { output } = await runC3({ argv });
expect(output).toEqual(version);
});
test("--version with positionals", async () => {
const argv = "foo bar baz --version".split(" ");
const { output } = await runC3({ argv });
expect(output).toEqual(version);
});

test("Using arrow keys + enter", async () => {
const { output } = await runC3({
argv: [projectPath],
promptHandlers: [
{
matcher: /What type of application do you want to create/,
input: [keys.enter],
},
{
matcher: /Do you want to use TypeScript/,
input: [keys.enter],
},
{
matcher: /Do you want to use git for version control/,
input: [keys.right, keys.enter],
},
{
matcher: /Do you want to deploy your application/,
input: [keys.left, keys.enter],
},
],
test("--version with flags", async () => {
const argv = "foo --type webFramework --no-deploy --version".split(" ");
const { output } = await runC3({ argv });
expect(output).toEqual(version);
});

expect(projectPath).toExist();
expect(output).toContain(`type "Hello World" Worker`);
expect(output).toContain(`yes typescript`);
expect(output).toContain(`no git`);
expect(output).toContain(`no deploy`);
});
test("Using arrow keys + enter", async () => {
const { output } = await runC3({
argv: [projectPath],
promptHandlers: [
{
matcher: /What type of application do you want to create/,
input: [keys.enter],
},
{
matcher: /Do you want to use TypeScript/,
input: [keys.enter],
},
{
matcher: /Do you want to use git for version control/,
input: [keys.right, keys.enter],
},
{
matcher: /Do you want to deploy your application/,
input: [keys.left, keys.enter],
},
],
});

test("Typing custom responses", async () => {
const { output } = await runC3({
argv: [],
promptHandlers: [
{
matcher: /In which directory do you want to create your application/,
input: [projectPath, keys.enter],
},
{
matcher: /What type of application do you want to create/,
input: [keys.down, keys.enter],
},
{
matcher: /Do you want to use TypeScript/,
input: ["n"],
},
{
matcher: /Do you want to use git for version control/,
input: ["n"],
},
{
matcher: /Do you want to deploy your application/,
input: ["n"],
},
],
expect(projectPath).toExist();
expect(output).toContain(`type "Hello World" Worker`);
expect(output).toContain(`yes typescript`);
expect(output).toContain(`no git`);
expect(output).toContain(`no deploy`);
});

expect(projectPath).toExist();
expect(output).toContain(`type Example router & proxy Worker`);
expect(output).toContain(`no typescript`);
expect(output).toContain(`no git`);
expect(output).toContain(`no deploy`);
});
test("Typing custom responses", async () => {
const { output } = await runC3({
argv: [],
promptHandlers: [
{
matcher:
/In which directory do you want to create your application/,
input: [projectPath, keys.enter],
},
{
matcher: /What type of application do you want to create/,
input: [keys.down, keys.enter],
},
{
matcher: /Do you want to use TypeScript/,
input: ["n"],
},
{
matcher: /Do you want to use git for version control/,
input: ["n"],
},
{
matcher: /Do you want to deploy your application/,
input: ["n"],
},
],
});

test("Mixed args and interactive", async () => {
const { output } = await runC3({
argv: [projectPath, "--ts", "--no-deploy"],
promptHandlers: [
{
matcher: /What type of application do you want to create/,
input: [keys.enter],
},
{
matcher: /Do you want to use git for version control/,
input: ["n"],
},
],
expect(projectPath).toExist();
expect(output).toContain(`type Example router & proxy Worker`);
expect(output).toContain(`no typescript`);
expect(output).toContain(`no git`);
expect(output).toContain(`no deploy`);
});

expect(projectPath).toExist();
expect(output).toContain(`type "Hello World" Worker`);
expect(output).toContain(`yes typescript`);
expect(output).toContain(`no git`);
expect(output).toContain(`no deploy`);
});
});
test("Mixed args and interactive", async () => {
const { output } = await runC3({
argv: [projectPath, "--ts", "--no-deploy"],
promptHandlers: [
{
matcher: /What type of application do you want to create/,
input: [keys.enter],
},
{
matcher: /Do you want to use git for version control/,
input: ["n"],
},
],
});

expect(projectPath).toExist();
expect(output).toContain(`type "Hello World" Worker`);
expect(output).toContain(`yes typescript`);
expect(output).toContain(`no git`);
expect(output).toContain(`no deploy`);
});
}
);
13 changes: 11 additions & 2 deletions packages/create-cloudflare/e2e-tests/helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import crypto from "node:crypto";
import { tmpdir } from "os";
import { join } from "path";
import { spawn } from "cross-spawn";
import { sleep } from "helpers/common";
import { spinnerFrames } from "helpers/interactive";
import type { SpinnerStyle } from "helpers/interactive";

Expand Down Expand Up @@ -30,6 +31,7 @@ export type RunnerConfig = {
promptHandlers?: PromptHandler[];
argv?: string[];
outputPrefix?: string;
quarantine?: boolean;
};

export const runC3 = async ({
Expand All @@ -46,14 +48,17 @@ export const runC3 = async ({
const lines: string[] = data.toString().split("\n");
const currentDialog = promptHandlers[0];

lines.forEach((line) => {
lines.forEach(async (line) => {
// Uncomment to debug test output
if (filterLine(line)) {
console.log(`${outputPrefix} ${line}`);
}
stdout.push(line);

if (currentDialog && currentDialog.matcher.test(line)) {
// Add a small sleep to avoid input race
await sleep(500);

currentDialog.input.forEach((keystroke) => {
proc.stdin.write(keystroke);
});
Expand Down Expand Up @@ -103,7 +108,7 @@ export const testProjectDir = (suite: string) => {
mkdtempSync(join(tmpdir(), `c3-tests-${suite}`))
);

const randomSuffix = crypto.randomBytes(3).toString("hex");
const randomSuffix = crypto.randomBytes(4).toString("hex");
const baseProjectName = `${C3_E2E_PREFIX}${randomSuffix}`;

const getName = (suffix: string) => `${baseProjectName}-${suffix}`;
Expand Down Expand Up @@ -136,3 +141,7 @@ const filterLine = (line: string) => {

return true;
};

export const isQuarantineMode = () => {
return process.env.E2E_QUARANTINE === "true";
};
Loading

0 comments on commit 975cbd9

Please sign in to comment.