feat(ai-help): add index with rendered macros

.github/workflows/prod-build.yml

@@ -259,16 +259,7 @@ jobs:
 
           yarn tool sync-translated-content
 
-          # Build using one process per locale.
-          # Note: We have 4 cores, but 9 processes is a reasonable number.
-          for locale in en-us es fr ja ko pt-br ru zh-cn zh-tw; do
-            yarn build --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
-            pids+=($!)
-          done
-
-          for pid in "${pids[@]}"; do
-            wait $pid
-          done
+          yarn build --locale en-us
 
           du -sh client/build
 
@@ -283,115 +274,9 @@ jobs:
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json
           yarn tool whatsdeployed $CONTENT_TRANSLATED_ROOT --output client/build/_whatsdeployed/translated-content.json
 
-      - name: Update search index
-        if: ${{ ! vars.SKIP_BUILD }}
+      - name: Update AI Help index with macros
+        run: yarn ai-help-macros update-index
         env:
-          DEPLOYER_ELASTICSEARCH_URL: ${{ secrets.DEPLOYER_PROD_ELASTICSEARCH_URL }}
-        run: |
-          cd deployer
-          poetry run deployer search-index ../client/build
-
-      - name: Authenticate with GCP
-        if: ${{ ! vars.SKIP_BUILD }}
-        uses: google-github-actions/auth@v2
-        with:
-          token_format: access_token
-          service_account: deploy-prod-content@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com
-          workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions
-
-      - name: Setup gcloud
-        if: ${{ ! vars.SKIP_BUILD }}
-        uses: google-github-actions/setup-gcloud@v1
-
-      - name: Sync build
-        if: ${{ ! vars.SKIP_BUILD }}
-        run: |-
-          gsutil -q -m -h "Cache-Control: public, max-age=3600" cp -r client/build/static gs://${{ vars.GCP_BUCKET_NAME }}/main/
-          gsutil -q -m -h "Cache-Control: public, max-age=3600" rsync -cdrj html,json,txt -y "^static/" client/build gs://${{ vars.GCP_BUCKET_NAME }}/main
-
-      - name: Authenticate with GCP
-        if: ${{ ! vars.SKIP_FUNCTION }}
-        uses: google-github-actions/auth@v2
-        with:
-          token_format: access_token
-          service_account: deploy-prod-prod-mdn-ingress@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com
-          workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions
-
-      - name: Setup gcloud
-        if: ${{ ! vars.SKIP_FUNCTION }}
-        uses: google-github-actions/setup-gcloud@v1
-        with:
-          install_components: "beta"
-
-      - name: Generate redirects map
-        if: ${{ ! vars.SKIP_FUNCTION }}
-        working-directory: cloud-function
-        env:
-          CONTENT_ROOT: ${{ github.workspace }}/mdn/content/files
-          CONTENT_TRANSLATED_ROOT: ${{ github.workspace }}/mdn/translated-content/files
-        run: |-
-          npm ci
-          npm run build-redirects
-
-      - name: Deploy Function
-        if: ${{ ! vars.SKIP_FUNCTION }}
-        run: |-
-          for region in europe-west1 us-west1 asia-east1; do
-            gcloud beta functions deploy mdn-prod-prod-$region \
-            --gen2 \
-            --runtime=nodejs18 \
-            --region=$region \
-            --source=cloud-function \
-            --trigger-http \
-            --allow-unauthenticated \
-            --entry-point=mdnHandler \
-            --concurrency=100 \
-            --min-instances=10 \
-            --max-instances=1000 \
-            --memory=2GB \
-            --timeout=60s \
-            --set-env-vars="ORIGIN_MAIN=developer.mozilla.org" \
-            --set-env-vars="ORIGIN_LIVE_SAMPLES=live.mdnplay.dev" \
-            --set-env-vars="ORIGIN_PLAY=mdnplay.dev" \
-            --set-env-vars="SOURCE_CONTENT=https://storage.googleapis.com/${{ vars.GCP_BUCKET_NAME }}/main/" \
-            --set-env-vars="SOURCE_API=https://api.developer.mozilla.org/" \
-            --set-env-vars="BSA_ENABLED=true" \
-            --set-env-vars="SENTRY_DSN=${{ secrets.SENTRY_DSN_CLOUD_FUNCTION }}" \
-            --set-env-vars="SENTRY_ENVIRONMENT=prod" \
-            --set-env-vars="SENTRY_TRACES_SAMPLE_RATE=${{ vars.SENTRY_TRACES_SAMPLE_RATE }}" \
-            --set-env-vars="SENTRY_RELEASE=${{ github.sha }}" \
-            --set-secrets="KEVEL_SITE_ID=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/prod-kevel-site-id/versions/latest" \
-            --set-secrets="KEVEL_NETWORK_ID=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/prod-kevel-network-id/versions/latest" \
-            --set-secrets="SIGN_SECRET=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/prod-sign-secret/versions/latest" \
-            --set-secrets="BSA_ZONE_KEYS=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/prod-bsa-zone-keys/versions/latest" \
-            2>&1 | sed "s/^/[$region] /" &
-            pids+=($!)
-          done
-
-          for pid in "${pids[@]}"; do
-            wait $pid
-          done
-
-      - name: Update AI Help index
-        run: yarn ai-help update-index
-        env:
-          CONTENT_ROOT: ${{ github.workspace }}/mdn/content/files
           OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
           SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
           SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
-
-      - name: Slack Notification
-        if: failure()
-        uses: rtCamp/action-slack-notify@v2
-        env:
-          SLACK_CHANNEL: mdn-notifications
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_ICON: https://avatars.slack-edge.com/2020-11-17/1513880588420_fedd7f0e9456888e69ff_96.png
-          SLACK_TITLE: ":rotating_light: Prod :rotating_light:"
-          SLACK_MESSAGE: "Build failed :collision:"
-          SLACK_FOOTER: "Powered by prod-build.yml"
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-
-      - name: Invalidate Google Cloud CDN
-        if: ${{ github.event.inputs.invalidate }}
-        run: gcloud compute url-maps invalidate-cdn-cache ${{ secrets.GCP_LOAD_BALANCER_NAME }} --path "/*" --async

.github/workflows/stage-build.yml

@@ -252,12 +252,7 @@ jobs:
 
           yarn tool sync-translated-content
 
-          # Build using one process per locale.
-          # Note: We have 4 cores, but 9 processes is a reasonable number.
-          for locale in en-us es fr ja ko pt-br ru zh-cn zh-tw; do
-            yarn build --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
-            pids+=($!)
-          done
+          yarn build --locale en-us
 
           for pid in "${pids[@]}"; do
             wait $pid
@@ -276,115 +271,9 @@ jobs:
           yarn tool whatsdeployed $CONTENT_ROOT --output client/build/_whatsdeployed/content.json
           yarn tool whatsdeployed $CONTENT_TRANSLATED_ROOT --output client/build/_whatsdeployed/translated-content.json
 
-      - name: Update search index
-        if: ${{ ! vars.SKIP_BUILD }}
-        env:
-          DEPLOYER_ELASTICSEARCH_URL: ${{ secrets.DEPLOYER_STAGE_ELASTICSEARCH_URL }}
-        run: |
-          cd deployer
-          poetry run deployer search-index ../client/build
-
-      - name: Authenticate with GCP
-        if: ${{ ! vars.SKIP_BUILD }}
-        uses: google-github-actions/auth@v2
-        with:
-          token_format: access_token
-          service_account: deploy-stage-content@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com
-          workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions
-
-      - name: Setup gcloud
-        if: ${{ ! vars.SKIP_BUILD }}
-        uses: google-github-actions/setup-gcloud@v1
-
-      - name: Sync build
-        if: ${{ ! vars.SKIP_BUILD }}
-        run: |-
-          gsutil -q -m -h "Cache-Control: public, max-age=3600" cp -r client/build/static gs://${{ vars.GCP_BUCKET_NAME }}/main/
-          gsutil -q -m -h "Cache-Control: public, max-age=3600" rsync -cdrj html,json,txt -y "^static/" client/build gs://${{ vars.GCP_BUCKET_NAME }}/main
-
-      - name: Authenticate with GCP
-        if: ${{ ! vars.SKIP_FUNCTION }}
-        uses: google-github-actions/auth@v2
-        with:
-          token_format: access_token
-          service_account: deploy-stage-nonprod-mdn-ingre@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com
-          workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions
-
-      - name: Setup gcloud
-        if: ${{ ! vars.SKIP_FUNCTION }}
-        uses: google-github-actions/setup-gcloud@v1
-        with:
-          install_components: "beta"
-
-      - name: Generate redirects map
-        if: ${{ ! vars.SKIP_FUNCTION }}
-        working-directory: cloud-function
+      - name: Update AI Help index with macros
+        run: yarn ai-help-macros update-index
         env:
-          CONTENT_ROOT: ${{ github.workspace }}/mdn/content/files
-          CONTENT_TRANSLATED_ROOT: ${{ github.workspace }}/mdn/translated-content/files
-        run: |
-          npm ci
-          npm run build-redirects
-
-      - name: Deploy Function
-        if: ${{ ! vars.SKIP_FUNCTION }}
-        run: |-
-          for region in europe-west1 us-west1 asia-east1; do
-            gcloud beta functions deploy mdn-nonprod-stage-$region \
-            --gen2 \
-            --runtime=nodejs18 \
-            --region=$region \
-            --source=cloud-function \
-            --trigger-http \
-            --allow-unauthenticated \
-            --entry-point=mdnHandler \
-            --concurrency=100 \
-            --min-instances=1 \
-            --max-instances=100 \
-            --memory=2GB \
-            --timeout=60s \
-            --set-env-vars="ORIGIN_MAIN=developer.allizom.org" \
-            --set-env-vars="ORIGIN_LIVE_SAMPLES=live.mdnyalp.dev" \
-            --set-env-vars="ORIGIN_PLAY=mdnyalp.dev" \
-            --set-env-vars="SOURCE_CONTENT=https://storage.googleapis.com/${{ vars.GCP_BUCKET_NAME }}/main/" \
-            --set-env-vars="SOURCE_API=https://api.developer.allizom.org/" \
-            --set-env-vars="BSA_ENABLED=true" \
-            --set-env-vars="SENTRY_DSN=${{ secrets.SENTRY_DSN_CLOUD_FUNCTION }}" \
-            --set-env-vars="SENTRY_ENVIRONMENT=stage" \
-            --set-env-vars="SENTRY_TRACES_SAMPLE_RATE=${{ vars.SENTRY_TRACES_SAMPLE_RATE }}" \
-            --set-env-vars="SENTRY_RELEASE=${{ github.sha }}" \
-            --set-secrets="KEVEL_SITE_ID=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/stage-kevel-site-id/versions/latest" \
-            --set-secrets="KEVEL_NETWORK_ID=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/stage-kevel-network-id/versions/latest" \
-            --set-secrets="SIGN_SECRET=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/stage-sign-secret/versions/latest" \
-            --set-secrets="BSA_ZONE_KEYS=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/stage-bsa-zone-keys/versions/latest" \
-            2>&1 | sed "s/^/[$region] /" &
-            pids+=($!)
-          done
-
-          for pid in "${pids[@]}"; do
-            wait $pid
-          done
-
-      - name: Update AI Help index
-        run: yarn ai-help update-index
-        env:
-          CONTENT_ROOT: ${{ github.workspace }}/mdn/content/files
           OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
           SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
           SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
-
-      - name: Slack Notification
-        if: failure()
-        uses: rtCamp/action-slack-notify@v2
-        env:
-          SLACK_CHANNEL: mdn-notifications
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_ICON: https://avatars.slack-edge.com/2020-11-17/1513880588420_fedd7f0e9456888e69ff_96.png
-          SLACK_TITLE: "Stage"
-          SLACK_MESSAGE: "Build failed :collision:"
-          SLACK_FOOTER: "Powered by stage-build.yml"
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-
-      - name: Invalidate CDN
-        if: ${{ github.event.inputs.invalidate }}
-        run: gcloud compute url-maps invalidate-cdn-cache ${{ secrets.GCP_LOAD_BALANCER_NAME }} --path "/*" --async

build/cli.ts

@@ -77,7 +77,11 @@ async function buildDocumentInteractive(
       );
     }
 
-    return { document, doc: await buildDocument(document), skip: false };
+    return {
+      document,
+      doc: await buildDocument(document, { plainHTML: true }),
+      skip: false,
+    };
   } catch (e) {
     if (!interactive) {
       throw e;
@@ -208,7 +212,7 @@ async function buildDocuments(
     }
 
     const {
-      doc: { doc: builtDocument, liveSamples, fileAttachmentMap },
+      doc: { doc: builtDocument, liveSamples, fileAttachmentMap, plainHTML },
       document,
     } = result;
 
@@ -230,6 +234,10 @@ async function buildDocuments(
       );
     }
 
+    if (plainHTML) {
+      fs.writeFileSync(path.join(outPath, "plain.html"), plainHTML);
+    }
+
     // This is exploiting the fact that renderHTML has the side-effect of
     // mutating the built document which makes this not great and refactor-worthy.
     const docString = JSON.stringify({ doc: builtDocument });

build/index.ts

@@ -164,13 +164,15 @@ export interface BuiltDocument {
   source?: {
     github_url: string;
   };
+  plainHTML?: string;
 }
 
 interface DocumentOptions {
   fixFlaws?: boolean;
   fixFlawsDryRun?: boolean;
   fixFlawsTypes?: Iterable<string>;
   fixFlawsVerbose?: boolean;
+  plainHTML?: boolean;
 }
 
 export async function buildDocument(
@@ -445,6 +447,12 @@ export async function buildDocument(
     throw error;
   }
 
+  // Dump HTML for GPT context
+  let plainHTML;
+  if (documentOptions.plainHTML) {
+    plainHTML = $.html();
+  }
+
   // Apply syntax highlighting all <pre> tags.
   syntaxHighlight($, doc);
 
@@ -555,7 +563,7 @@ export async function buildDocument(
     document.metadata.slug.startsWith("orphaned/") ||
     document.metadata.slug.startsWith("conflicting/");
 
-  return { doc: doc as Doc, liveSamples, fileAttachmentMap };
+  return { doc: doc as Doc, liveSamples, fileAttachmentMap, plainHTML };
 }
 
 function addBaseline(doc: Partial<Doc>) {

build/resolve-bcd.ts

@@ -1,10 +1,10 @@
 import bcdUntyped from "@mdn/browser-compat-data/forLegacyNode";
-import { CompatData } from "@mdn/browser-compat-data/types";
+import { CompatData, Identifier } from "@mdn/browser-compat-data/types";
 
 const bcd = bcdUntyped as CompatData;
 
 export function packageBCD(query) {
-  const data = query.split(".").reduce((prev, curr) => {
+  const data: Identifier = query.split(".").reduce((prev, curr) => {
     return prev && Object.prototype.hasOwnProperty.call(prev, curr)
       ? prev[curr]
       : undefined;

package.json

@@ -14,6 +14,7 @@
   },
   "scripts": {
     "ai-help": "ts-node scripts/ai-help.ts",
+    "ai-help-macros": "ts-node scripts/ai-help-macros.ts",
     "analyze": "source-map-explorer 'client/build/static/js/*.js'",
     "analyze:css": "source-map-explorer 'client/build/static/css/*.css'",
     "build": "cross-env NODE_ENV=production NODE_OPTIONS='--no-warnings=ExperimentalWarning --loader ts-node/esm' node build/cli.ts",