diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 20f76b3a415c..023f518411dd 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -159,6 +159,9 @@ jobs: # Generate sitemap index file yarn build --sitemap-index + # SSR all pages + yarn ssr + # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml index 3f86b098a267..b70ccf0f7531 100644 --- a/.github/workflows/stage-build.yml +++ b/.github/workflows/stage-build.yml @@ -306,6 +306,9 @@ jobs: # Build the curriculum yarn build:curriculum + # SSR all pages + yarn ssr + # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json diff --git a/.github/workflows/xyz-build.yml b/.github/workflows/xyz-build.yml index a9550e925b2e..edb256ef09cc 100644 --- a/.github/workflows/xyz-build.yml +++ b/.github/workflows/xyz-build.yml @@ -190,6 +190,9 @@ jobs: # Build the curriculum yarn build:curriculum + # SSR all pages + yarn ssr + # Generate whatsdeployed files. yarn tool whatsdeployed --output client/build/_whatsdeployed/code.json yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json diff --git a/build/blog.ts b/build/blog.ts index b6c22ef8b9cf..f450992d2b4d 100644 --- a/build/blog.ts +++ b/build/blog.ts @@ -18,9 +18,6 @@ import { AuthorFrontmatter, AuthorMetadata, } from "../libs/types/blog.js"; -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore -import { renderHTML } from "../ssr/dist/main.js"; import { findPostFileBySlug, injectLoadingLazyAttributes, @@ -242,21 +239,22 @@ export async function buildBlogIndex(options: { verbose?: boolean }) { const hyData = { posts: await allPostFrontmatter(), }; - const context = { hyData, pageTitle: "MDN Blog" }; + const context = { + hyData, + pageTitle: "MDN Blog", + url: `/${locale}/${prefix}/`, + }; - const html = renderHTML(`/${locale}/${prefix}/`, context); const outPath = path.join(BUILD_OUT_ROOT, locale.toLowerCase(), `${prefix}`); await fs.mkdir(outPath, { recursive: true }); - const filePath = path.join(outPath, "index.html"); const jsonFilePath = path.join(outPath, "index.json"); await fs.mkdir(outPath, { recursive: true }); - await fs.writeFile(filePath, html); await fs.writeFile(jsonFilePath, JSON.stringify(context)); if (options.verbose) { - console.log("Wrote", filePath); + console.log("Wrote", jsonFilePath); } } @@ -279,8 +277,8 @@ export async function buildBlogPosts(options: { continue; } - const url = `/${locale}/blog/${blogMeta.slug}/`; - const renderUrl = `/${locale}/blog/${blogMeta.slug}`; + const url = `/${locale}/${prefix}/${blogMeta.slug}/`; + const renderUrl = `/${locale}/${prefix}/${blogMeta.slug}`; const renderDoc: BlogPostDoc = { url: renderUrl, rawBody: body, @@ -302,6 +300,7 @@ export async function buildBlogPosts(options: { locale, noIndexing: options.noIndexing, image: blogMeta.image?.file && `${BASE_URL}${url}${blogMeta.image?.file}`, + url, }; const outPath = path.join( @@ -329,17 +328,13 @@ export async function buildBlogPosts(options: { await fs.copyFile(from, to); } - const html = renderHTML(`/${locale}/${prefix}/${blogMeta.slug}/`, context); - - const filePath = path.join(outPath, "index.html"); const jsonFilePath = path.join(outPath, "index.json"); await fs.mkdir(outPath, { recursive: true }); - await fs.writeFile(filePath, html); await fs.writeFile(jsonFilePath, JSON.stringify(context)); if (options.verbose) { - console.log("Wrote", filePath); + console.log("Wrote", jsonFilePath); } } } diff --git a/build/cli.ts b/build/cli.ts index dff23999ed30..f4b9526a301d 100644 --- a/build/cli.ts +++ b/build/cli.ts @@ -19,7 +19,6 @@ import { import { DEFAULT_LOCALE, VALID_LOCALES } from "../libs/constants/index.js"; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore -import { renderHTML } from "../ssr/dist/main.js"; import options from "./build-options.js"; import { buildDocument, @@ -32,6 +31,7 @@ import { makeSitemapXML, makeSitemapIndexXML } from "./sitemaps.js"; import { humanFileSize } from "./utils.js"; import { initSentry } from "./sentry.js"; import { macroRenderTimes } from "../kumascript/src/render.js"; +import { ssrAllDocuments } from "./ssr.js"; const { program } = caporal; const { prompt } = inquirer; @@ -129,7 +129,6 @@ async function buildDocuments( files: string[] = null, quiet = false, interactive = false, - noHTML = false, locales: Map = new Map() ): Promise { // If a list of files was set, it came from the CLI. @@ -230,20 +229,16 @@ async function buildDocuments( updateBaselineBuildMetadata(builtDocument); } - if (!noHTML) { - fs.writeFileSync( - path.join(outPath, "index.html"), - renderHTML(document.url, { doc: builtDocument }) - ); - } - if (plainHTML) { fs.writeFileSync(path.join(outPath, "plain.html"), plainHTML); } // This is exploiting the fact that renderHTML has the side-effect of // mutating the built document which makes this not great and refactor-worthy. - const docString = JSON.stringify({ doc: builtDocument }); + const docString = JSON.stringify({ + doc: builtDocument, + url: builtDocument.mdn_url, + }); fs.writeFileSync(path.join(outPath, "index.json"), docString); fs.writeFileSync( path.join(outPath, "contributors.txt"), @@ -446,7 +441,6 @@ interface BuildArgsAndOptions { options: { quiet?: boolean; interactive?: boolean; - nohtml?: boolean; locale?: string[]; notLocale?: string[]; sitemapIndex?: boolean; @@ -459,12 +453,10 @@ if (SENTRY_DSN_BUILD) { program .name("build") + .command("build", "build content") .option("-i, --interactive", "Ask what to do when encountering flaws", { default: false, }) - .option("-n, --nohtml", "Do not build index.html", { - default: false, - }) .option("-l, --locale ", "Filtered specific locales", { default: [], validator: [...VALID_LOCALES.keys()], @@ -567,7 +559,6 @@ program files, Boolean(options.quiet), Boolean(options.interactive), - Boolean(options.nohtml), locales ); const t1 = new Date(); @@ -607,6 +598,10 @@ program } }); +program.command("render", "render all documents").action(async () => { + await ssrAllDocuments(); +}); + program.run(); function compareBigInt(a: bigint, b: bigint): number { if (a < b) { diff --git a/build/curriculum.ts b/build/curriculum.ts index 6d66343c2136..5fe37537827f 100644 --- a/build/curriculum.ts +++ b/build/curriculum.ts @@ -37,7 +37,6 @@ import { HydrationData } from "../libs/types/hydration.js"; import { memoize, slugToFolder } from "../content/utils.js"; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore -import { renderHTML } from "../ssr/dist/main.js"; import { CheerioAPI } from "cheerio"; export const allFiles = memoize(async () => { @@ -367,6 +366,7 @@ export async function buildCurriculum(options: { pageTitle: meta.title, locale, noIndexing: options.noIndexing, + url: `/${locale}/${meta.slug}/`, }; const outPath = path.join( @@ -377,17 +377,13 @@ export async function buildCurriculum(options: { await fs.mkdir(outPath, { recursive: true }); - const html: string = renderHTML(`/${locale}/${meta.slug}/`, context); - - const filePath = path.join(outPath, "index.html"); const jsonFilePath = path.join(outPath, "index.json"); await fs.mkdir(outPath, { recursive: true }); - await fs.writeFile(filePath, html); await fs.writeFile(jsonFilePath, JSON.stringify(context)); if (options.verbose) { - console.log("Wrote", filePath); + console.log("Wrote", jsonFilePath); } } } diff --git a/build/spas.ts b/build/spas.ts index 1afb175a51a5..e9e09cb398f7 100644 --- a/build/spas.ts +++ b/build/spas.ts @@ -22,9 +22,6 @@ import { } from "../libs/env/index.js"; import { isValidLocale } from "../libs/locale-utils/index.js"; import { DocFrontmatter, DocParent, NewsItem } from "../libs/types/document.js"; -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore -import { renderHTML } from "../ssr/dist/main.js"; import { getSlugByBlogPostUrl, splitSections } from "./utils.js"; import { findByURL } from "../content/document.js"; import { buildDocument } from "./index.js"; @@ -73,26 +70,23 @@ async function buildContributorSpotlight( usernames: frontMatter.attributes.usernames, quote: frontMatter.attributes.quote, }; - const context = { hyData }; + const context = { hyData, url: `/${locale}/${prefix}/${contributor}` }; - const html = renderHTML(`/${locale}/${prefix}/${contributor}`, context); const outPath = path.join( BUILD_OUT_ROOT, locale.toLowerCase(), `${prefix}/${hyData.folderName}` ); - const filePath = path.join(outPath, "index.html"); const imgFilePath = `${contributorSpotlightRoot}/${contributor}/profile-image.jpg`; const imgFileDestPath = path.join(outPath, profileImg); const jsonFilePath = path.join(outPath, "index.json"); fs.mkdirSync(outPath, { recursive: true }); - fs.writeFileSync(filePath, html); fs.copyFileSync(imgFilePath, imgFileDestPath); fs.writeFileSync(jsonFilePath, JSON.stringify(context)); if (options.verbose) { - console.log("Wrote", filePath); + console.log("Wrote", jsonFilePath); } if (frontMatter.attributes.is_featured) { return { @@ -112,17 +106,21 @@ export async function buildSPAs(options: { // The URL isn't very important as long as it triggers the right route in the const url = `/${DEFAULT_LOCALE}/404.html`; - const html = renderHTML(url, { pageNotFound: true }); + const context = { url, pageNotFound: true }; const outPath = path.join( BUILD_OUT_ROOT, DEFAULT_LOCALE.toLowerCase(), "_spas" ); fs.mkdirSync(outPath, { recursive: true }); - fs.writeFileSync(path.join(outPath, path.basename(url)), html); + const jsonFilePath = path.join( + outPath, + path.basename(url).replace(/\.html$/, ".json") + ); + fs.writeFileSync(jsonFilePath, JSON.stringify(context)); buildCount++; if (options.verbose) { - console.log("Wrote", path.join(outPath, path.basename(url))); + console.log("Wrote", jsonFilePath); } // Basically, this builds one (for example) `search/index.html` for every @@ -183,16 +181,16 @@ export async function buildSPAs(options: { pageTitle, locale, noIndexing, + url, }; - const html = renderHTML(url, context); const outPath = path.join(BUILD_OUT_ROOT, pathLocale, prefix); fs.mkdirSync(outPath, { recursive: true }); - const filePath = path.join(outPath, "index.html"); - fs.writeFileSync(filePath, html); + const jsonFilePath = path.join(outPath, "index.json"); + fs.writeFileSync(jsonFilePath, JSON.stringify(context)); buildCount++; if (options.verbose) { - console.log("Wrote", filePath); + console.log("Wrote", jsonFilePath); } } } @@ -240,9 +238,9 @@ export async function buildSPAs(options: { const context = { hyData, pageTitle: `${frontMatter.attributes.title || ""} | ${title}`, + url, }; - const html = renderHTML(url, context); const outPath = path.join( BUILD_OUT_ROOT, locale, @@ -250,14 +248,12 @@ export async function buildSPAs(options: { page ); fs.mkdirSync(outPath, { recursive: true }); - const filePath = path.join(outPath, "index.html"); - fs.writeFileSync(filePath, html); + const jsonFilePath = path.join(outPath, "index.json"); + fs.writeFileSync(jsonFilePath, JSON.stringify(context)); buildCount++; if (options.verbose) { - console.log("Wrote", filePath); + console.log("Wrote", jsonFilePath); } - const filePathContext = path.join(outPath, "index.json"); - fs.writeFileSync(filePathContext, JSON.stringify(context)); } } @@ -341,24 +337,15 @@ export async function buildSPAs(options: { latestNews, featuredArticles, }; - const context = { hyData }; - const html = renderHTML(url, context); + const context = { hyData, url }; const outPath = path.join(BUILD_OUT_ROOT, localeLC); fs.mkdirSync(outPath, { recursive: true }); - const filePath = path.join(outPath, "index.html"); - fs.writeFileSync(filePath, html); - buildCount++; - if (options.verbose) { - console.log("Wrote", filePath); - } - // Also, dump the recent pull requests in a file so the data can be gotten - // in client-side rendering. - const filePathContext = path.join(outPath, "index.json"); - fs.writeFileSync(filePathContext, JSON.stringify(context)); + const jsonFilePath = path.join(outPath, "index.json"); + fs.writeFileSync(jsonFilePath, JSON.stringify(context)); buildCount++; if (options.verbose) { - console.log("Wrote", filePathContext); + console.log("Wrote", jsonFilePath); } } } diff --git a/build/ssr.ts b/build/ssr.ts new file mode 100644 index 000000000000..8e6bf0f8e128 --- /dev/null +++ b/build/ssr.ts @@ -0,0 +1,53 @@ +import { fdir } from "fdir"; +import { BUILD_OUT_ROOT } from "../libs/env/index.js"; +import { readFile, writeFile } from "node:fs/promises"; +import { renderHTML } from "../ssr/dist/main.js"; +import { HydrationData } from "../libs/types/hydration.js"; + +export async function ssrAllDocuments() { + const api = new fdir() + .withFullPaths() + .withErrors() + .filter( + (filePath) => + filePath.endsWith("index.json") || filePath.endsWith("404.json") + ) + .crawl(BUILD_OUT_ROOT); + const docs = await api.withPromise(); + + const t0 = new Date(); + + const done = []; + for (let i = 0; i < docs.length; i += 1000) { + const chunk = docs.slice(i, i + 1000); + const out = await Promise.all( + chunk + .map(async (file) => { + const context: HydrationData = JSON.parse( + await readFile(file, "utf-8") + ); + if (!context?.url) { + return null; + } + const html = renderHTML(context); + const outputFile = file.replace(/.json$/, ".html"); + await writeFile(outputFile, html); + return outputFile; + }) + .filter(Boolean) + ); + done.push(...out); + } + const t1 = new Date(); + const count = done.length; + const seconds = (t1.getTime() - t0.getTime()) / 1000; + const took = + seconds > 60 + ? `${(seconds / 60).toFixed(1)} minutes` + : `${seconds.toFixed(1)} seconds`; + console.log( + `Rendered ${count.toLocaleString()} pages in ${took}, at a rate of ${( + count / seconds + ).toFixed(1)} documents per second.` + ); +} diff --git a/client/src/app.test.tsx b/client/src/app.test.tsx index e87662a38577..3040eec88b75 100644 --- a/client/src/app.test.tsx +++ b/client/src/app.test.tsx @@ -6,7 +6,7 @@ import { App } from "./app"; it("renders without crashing", () => { const app = ( - + ); const div = document.createElement("div"); diff --git a/libs/types/hydration.ts b/libs/types/hydration.ts index 454675a941d4..63c014b60c55 100644 --- a/libs/types/hydration.ts +++ b/libs/types/hydration.ts @@ -10,6 +10,7 @@ interface HydrationData { locale?: any; noIndexing?: any; image?: string | null; + url: string; } export type { HydrationData }; diff --git a/package.json b/package.json index b4d40be2d363..f9aba05ba373 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "ai-help-macros": "cross-env NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node scripts/ai-help-macros.ts", "analyze": "source-map-explorer 'client/build/static/js/*.js'", "analyze:css": "source-map-explorer 'client/build/static/css/*.css'", - "build": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/cli.ts", + "build": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/cli.ts build", "build:blog": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/build-blog.ts", "build:client": "cd client && cross-env NODE_ENV=production BABEL_ENV=production INLINE_RUNTIME_CHUNK=false node scripts/build.js", "build:curriculum": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/build-curriculum.ts", @@ -38,6 +38,7 @@ "prepare": "(husky || true) && yarn install:all && yarn install:all:npm", "prettier-check": "prettier --check .", "prettier-format": "prettier --write .", + "ssr": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/cli.ts render", "start": "(test -f client/build/index.html || yarn build:client) && (test -f ssr/dist/main.js || yarn build:ssr) && (test -f popularities.json || yarn tool popularities) && (test -d client/build/en-us/_spas || yarn tool spas) && nf -j Procfile.start start", "start:client": "cd client && cross-env NODE_ENV=development BABEL_ENV=development BROWSER=none PORT=3000 node scripts/start.js", "start:server": "node-dev --experimental-loader ts-node/esm server/index.ts", diff --git a/server/index.ts b/server/index.ts index 4d1178388f1c..fd5fe42929c2 100644 --- a/server/index.ts +++ b/server/index.ts @@ -448,7 +448,7 @@ app.get("/*", async (req, res, ...args) => { res.json({ doc: document }); } else { res.header("Content-Security-Policy", CSP_VALUE); - res.send(renderHTML(lookupURL, { doc: document })); + res.send(renderHTML({ doc: document, url: lookupURL })); } }); diff --git a/ssr/index.ts b/ssr/index.ts index 0322a0228441..60b1b81623bf 100644 --- a/ssr/index.ts +++ b/ssr/index.ts @@ -3,14 +3,16 @@ import { StaticRouter } from "react-router-dom/server"; import { App } from "../client/src/app"; import render from "./render"; +import { HydrationData } from "../libs/types/hydration"; -export function renderHTML(url, context) { +export function renderHTML(context: HydrationData) { return render( React.createElement( StaticRouter, - { location: url }, + { location: context.url }, React.createElement(App, context) ), + context.url, context ); } diff --git a/ssr/render.ts b/ssr/render.ts index 12f9d27c8624..e0a6175166da 100644 --- a/ssr/render.ts +++ b/ssr/render.ts @@ -99,6 +99,7 @@ const readBuildHTML = lazy(() => { export default function render( renderApp, + url: string, { doc = null, pageNotFound = false, @@ -109,7 +110,7 @@ export default function render( noIndexing = null, image = null, blogMeta = null, - }: HydrationData = {} + }: HydrationData = { url } ) { const buildHtml = readBuildHTML(); const rendered = renderToString(renderApp); @@ -119,7 +120,7 @@ export default function render( let pageDescription = ""; let escapedPageTitle = htmlEscape(pageTitle); - const hydrationData: HydrationData = {}; + const hydrationData: HydrationData = { url }; const translations: string[] = []; if (blogMeta) { hydrationData.blogMeta = blogMeta;