diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 5084e41bcb69..bbf8daf398d9 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -141,9 +141,9 @@ jobs: # Spread the work across 2 processes. Why 2? Because that's what you # get in the default GitHub hosting Linux runners. # See https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources - yarn build --locale en-us --locale ja --locale fr & + yarn build:json --locale en-us --locale ja --locale fr & build1=$! - yarn build --not-locale en-us --not-locale ja --not-locale fr & + yarn build:json --not-locale en-us --not-locale ja --not-locale fr & build2=$! # You must explicitly specify the job you're waiting-on to ensure @@ -160,7 +160,7 @@ jobs: yarn build --sitemap-index # SSR all pages - yarn build:render + yarn build:render-html # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index b70085f1a4da..326a1ba2b66e 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -53,9 +53,9 @@ jobs: BUILD_GOOGLE_ANALYTICS_MEASUREMENT_ID: G-XXXXXXXX run: | yarn build:prepare - # BUILD_FOLDERSEARCH=mdn/kitchensink yarn build - BUILD_FOLDERSEARCH=web/javascript/reference/global_objects/array/foreach yarn build - yarn build:render + # BUILD_FOLDERSEARCH=mdn/kitchensink yarn build:json + BUILD_FOLDERSEARCH=web/javascript/reference/global_objects/array/foreach yarn build:json + yarn build:render-html - name: Serve and lhci env: diff --git a/.github/workflows/prod-build.yml b/.github/workflows/prod-build.yml index 9d2e92a31049..6196facd2699 100644 --- a/.github/workflows/prod-build.yml +++ b/.github/workflows/prod-build.yml @@ -264,7 +264,7 @@ jobs: # Build using one process per locale. # Note: We have 4 cores, but 9 processes is a reasonable number. for locale in en-us es fr ja ko pt-br ru zh-cn zh-tw; do - yarn build --locale $locale 2>&1 | sed "s/^/[$locale] /" & + yarn build:json --locale $locale 2>&1 | sed "s/^/[$locale] /" & pids+=($!) done @@ -284,7 +284,7 @@ jobs: yarn build:curriculum # SSR all pages - yarn build:render + yarn build:render-html # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml index 85bcdfec5c23..aa22e02255af 100644 --- a/.github/workflows/stage-build.yml +++ b/.github/workflows/stage-build.yml @@ -280,7 +280,7 @@ jobs: # Build using one process per locale. # Note: We have 4 cores, but 9 processes is a reasonable number. for locale in en-us es fr ja ko pt-br ru zh-cn zh-tw; do - yarn build --locale $locale 2>&1 | sed "s/^/[$locale] /" & + yarn build:json --locale $locale 2>&1 | sed "s/^/[$locale] /" & pids+=($!) done @@ -300,7 +300,7 @@ jobs: yarn build:curriculum # SSR all pages - yarn build:render + yarn build:render-html # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 40c6f94608e2..e57696a8b4fa 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -90,8 +90,8 @@ jobs: ENV_FILE: .env.testing run: | yarn build:prepare - yarn build - yarn build:render + yarn build:json + yarn build:render-html yarn start:static-server > /tmp/stdout.log 2> /tmp/stderr.log & sleep 1 diff --git a/.github/workflows/xyz-build.yml b/.github/workflows/xyz-build.yml index 800db4b75581..978e92132cd6 100644 --- a/.github/workflows/xyz-build.yml +++ b/.github/workflows/xyz-build.yml @@ -171,7 +171,7 @@ jobs: # Build using one process per locale. # Note: We have 4 cores, but 9 processes is a reasonable number. for locale in en-us es fr ja ko pt-br ru zh-cn zh-tw; do - yarn build --locale $locale 2>&1 | sed "s/^/[$locale] /" & + yarn build:json --locale $locale 2>&1 | sed "s/^/[$locale] /" & pids+=($!) done @@ -191,7 +191,7 @@ jobs: yarn build:curriculum # SSR all pages - yarn build:render + yarn build:render-html # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json diff --git a/README.md b/README.md index 4cc12bd2cf42..252f6c4a051c 100644 --- a/README.md +++ b/README.md @@ -221,18 +221,17 @@ pages, but you can pre-emptively build all the content in advance if desired. One potential advantage is that you can get a more complete list of all possible "flaws" across all documents before you even visit them. -The two most fundamental CLI commands are: +The most fundamental CLI command is: yarn build - yarn build:render ### What gets built -`yarn build` builds every `index.md` into an `index.json` which contains all -input data for our React client to render a page. +Every `index.html` becomes two files: -`yarn build:render` renders every of the previously built `index.json` into a -`index.html` which contains the fully formed and complete HTML for the page. +- `index.html` — a fully formed and complete HTML file +- `index.json` — the state information React needs to build the page in the + client ### Flaw checks diff --git a/build/cli.ts b/build/cli.ts index 866935e3725b..f4e83c75eae3 100644 --- a/build/cli.ts +++ b/build/cli.ts @@ -31,7 +31,8 @@ import { makeSitemapXML, makeSitemapIndexXML } from "./sitemaps.js"; import { humanFileSize } from "./utils.js"; import { initSentry } from "./sentry.js"; import { macroRenderTimes } from "../kumascript/src/render.js"; -import { ssrAllDocuments } from "./ssr.js"; +import { ssrAllDocuments, ssrDocument } from "./ssr.js"; +import { HydrationData } from "../libs/types/hydration.js"; const { program } = caporal; const { prompt } = inquirer; @@ -139,6 +140,7 @@ async function buildDocuments( files: string[] = null, quiet = false, interactive = false, + noHTML = false, locales: Map = new Map() ): Promise { // If a list of files was set, it came from the CLI. @@ -239,16 +241,21 @@ async function buildDocuments( updateBaselineBuildMetadata(builtDocument); } + const context: HydrationData = { + doc: builtDocument, + url: builtDocument.mdn_url, + }; + + if (!noHTML) { + fs.writeFileSync(path.join(outPath, "index.html"), ssrDocument(context)); + } if (plainHTML) { fs.writeFileSync(path.join(outPath, "plain.html"), plainHTML); } // This is exploiting the fact that renderHTML has the side-effect of // mutating the built document which makes this not great and refactor-worthy. - const docString = JSON.stringify({ - doc: builtDocument, - url: builtDocument.mdn_url, - }); + const docString = JSON.stringify(context); fs.writeFileSync(path.join(outPath, "index.json"), docString); fs.writeFileSync( path.join(outPath, "contributors.txt"), @@ -458,6 +465,7 @@ interface BuildArgsAndOptions { options: { quiet?: boolean; interactive?: boolean; + nohtml?: boolean; locale?: string[]; notLocale?: string[]; sitemapIndex?: boolean; @@ -474,6 +482,9 @@ program .option("-i, --interactive", "Ask what to do when encountering flaws", { default: false, }) + .option("-n, --nohtml", "Do not build index.html", { + default: false, + }) .option("-l, --locale ", "Filtered specific locales", { default: [], validator: [...VALID_LOCALES.keys()], @@ -576,6 +587,7 @@ program files, Boolean(options.quiet), Boolean(options.interactive), + Boolean(options.nohtml), locales ); const t1 = new Date(); @@ -615,9 +627,14 @@ program } }); -program.command("render", "render all documents").action(async () => { - await ssrAllDocuments(); -}); +program + .command("render", "render all documents") + .option("-n, --no-docs", "Do not build docs (only spas, blog...)", { + default: false, + }) + .action(async ({ options }) => { + await ssrAllDocuments(Boolean(options?.noDocs)); + }); program.run(); function compareBigInt(a: bigint, b: bigint): number { diff --git a/build/ssr.ts b/build/ssr.ts index a5f2ad1b8407..93005e1d81d7 100644 --- a/build/ssr.ts +++ b/build/ssr.ts @@ -6,10 +6,15 @@ import { readFile, writeFile } from "node:fs/promises"; import { renderHTML } from "../ssr/dist/main.js"; import { HydrationData } from "../libs/types/hydration.js"; -export async function ssrAllDocuments() { +export function ssrDocument(context: HydrationData) { + return renderHTML(context); +} + +export async function ssrAllDocuments(noDocs = false) { const api = new fdir() .withFullPaths() .withErrors() + .exclude((dirName) => noDocs && dirName === "docs") .filter( (filePath) => filePath.endsWith("index.json") || filePath.endsWith("404.json") diff --git a/package.json b/package.json index fdfaa44d9fe2..7f7d1a854a89 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,9 @@ "build:curriculum": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/build-curriculum.ts", "build:dist": "tsc -p tsconfig.dist.json", "build:glean": "cd client && cross-env VIRTUAL_ENV=venv glean translate src/telemetry/metrics.yaml src/telemetry/pings.yaml -f typescript -o src/telemetry/generated", + "build:json": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/cli.ts build -n", "build:prepare": "yarn build:client && yarn build:ssr && yarn tool popularities && yarn tool spas && yarn tool gather-git-history && yarn tool build-robots-txt", - "build:render": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/cli.ts render", + "build:render-html": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/cli.ts render", "build:ssr": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node ssr/prepare.ts && cd ssr && webpack --mode=production", "build:sw": "cd client/pwa && yarn && yarn build:prod", "build:sw-dev": "cd client/pwa && yarn && yarn build", @@ -52,7 +53,7 @@ "test:headless": "playwright test headless", "test:kumascript": "yarn jest --rootDir kumascript --env=node", "test:libs": "yarn jest --rootDir libs --env=node", - "test:prepare": "yarn build:prepare && yarn build && yarn start:static-server", + "test:prepare": "yarn build:prepare && yarn build:json && yarn build:render-html && yarn start:static-server", "test:testing": "yarn jest --rootDir testing", "tool": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node ./tool/cli.ts", "watch:ssr": "cd ssr && webpack --mode=production --watch" diff --git a/scripts/testing.sh b/scripts/testing.sh index ed6b6ce6d42e..6c7055a8b15d 100755 --- a/scripts/testing.sh +++ b/scripts/testing.sh @@ -23,8 +23,8 @@ echo "----------------------" export ENV_FILE=".env.testing" yarn build:prepare -yarn build -yarn build:render +yarn build:json +yarn build:render-html nohup yarn start:static-server > testing.log 2>&1 & PID=$! diff --git a/testing/README.md b/testing/README.md index 4159de4607b1..fe2ba009da37 100644 --- a/testing/README.md +++ b/testing/README.md @@ -21,8 +21,8 @@ To run these tests, first run: ```sh export ENV_FILE=.env.testing yarn build:prepare -yarn build -yarn build:render +yarn build:json +yarn build:render-html yarn start:static-server ``` diff --git a/testing/scripts/functional-test.sh b/testing/scripts/functional-test.sh index 30589c649ed4..ab784b3b6f4c 100755 --- a/testing/scripts/functional-test.sh +++ b/testing/scripts/functional-test.sh @@ -4,7 +4,7 @@ set -e export ENV_FILE=.env.testing yarn build:prepare -yarn build -yarn build:render +yarn build:json +yarn build:render-html yarn test:testing $@