fix(core): move resolving plugins back to main thread (#29176)

<!-- Please make sure you have read the submission guidelines before
posting an PR -->
<!--
https://github.com/nrwl/nx/blob/master/CONTRIBUTING.md#-submitting-a-pr
-->

<!-- Please make sure that your commit message follows our format -->
<!-- Example: `fix(nx): must begin with lowercase` -->

<!-- If this is a particularly complex change or feature addition, you
can request a dedicated Nx release for this pull request branch. Mention
someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they
will confirm if the PR warrants its own release for testing purposes,
and generate it for you if appropriate. -->

## Current Behavior
<!-- This is the behavior we have today -->

Resolving custom plugins to their paths to be loaded involves loading
our default plugins. When plugin isolation, this loads the default
plugins for each and every plugin worker.

## Expected Behavior
<!-- This is the behavior we should expect with the changes in this PR
-->

Resolution of the plugins is the same and easily serializable to send to
the worker. Resolving plugins on the main thread allows Nx to reuse the
default plugins and decrease the memory usage greatly.

## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #

packages/nx/src/project-graph/plugins/get-plugins.ts

@@ -5,6 +5,9 @@ import { workspaceRoot } from '../../utils/workspace-root';
 
 let currentPluginsConfigurationHash: string;
 let loadedPlugins: LoadedNxPlugin[];
+let pendingPluginsPromise:
+  | Promise<readonly [LoadedNxPlugin[], () => void]>
+  | undefined;
 let cleanup: () => void;
 
 export async function getPlugins() {
@@ -24,18 +27,20 @@ export async function getPlugins() {
     cleanup();
   }
 
+  pendingPluginsPromise ??= loadNxPlugins(pluginsConfiguration, workspaceRoot);
+
   currentPluginsConfigurationHash = pluginsConfigurationHash;
-  const [result, cleanupFn] = await loadNxPlugins(
-    pluginsConfiguration,
-    workspaceRoot
-  );
+  const [result, cleanupFn] = await pendingPluginsPromise;
   cleanup = cleanupFn;
   loadedPlugins = result;
   return result;
 }
 
 let loadedDefaultPlugins: LoadedNxPlugin[];
 let cleanupDefaultPlugins: () => void;
+let pendingDefaultPluginPromise:
+  | Promise<readonly [LoadedNxPlugin[], () => void]>
+  | undefined;
 
 export async function getOnlyDefaultPlugins() {
   // If the plugins configuration has not changed, reuse the current plugins
@@ -48,13 +53,17 @@ export async function getOnlyDefaultPlugins() {
     cleanupDefaultPlugins();
   }
 
-  const [result, cleanupFn] = await loadNxPlugins([], workspaceRoot);
+  pendingDefaultPluginPromise ??= loadNxPlugins([], workspaceRoot);
+
+  const [result, cleanupFn] = await pendingDefaultPluginPromise;
   cleanupDefaultPlugins = cleanupFn;
   loadedPlugins = result;
   return result;
 }
 
 export function cleanupPlugins() {
+  pendingPluginsPromise = undefined;
+  pendingDefaultPluginPromise = undefined;
   cleanup();
   cleanupDefaultPlugins();
 }

packages/nx/src/project-graph/plugins/isolation/messaging.ts

@@ -14,6 +14,9 @@ export interface PluginWorkerLoadMessage {
   payload: {
     plugin: PluginConfiguration;
     root: string;
+    name: string;
+    pluginPath: string;
+    shouldRegisterTSTranspiler: boolean;
   };
 }
 

packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts

@@ -16,6 +16,8 @@ import {
   isPluginWorkerResult,
   sendMessageOverSocket,
 } from './messaging';
+import { getNxRequirePaths } from '../../../utils/installation-directory';
+import { resolveNxPlugin } from '../loader';
 
 const cleanupFunctions = new Set<() => void>();
 
@@ -59,6 +61,10 @@ export async function loadRemoteNxPlugin(
   if (nxPluginWorkerCache.has(cacheKey)) {
     return [nxPluginWorkerCache.get(cacheKey), () => {}];
   }
+  const moduleName = typeof plugin === 'string' ? plugin : plugin.plugin;
+
+  const { name, pluginPath, shouldRegisterTSTranspiler } =
+    await resolveNxPlugin(moduleName, root, getNxRequirePaths(root));
 
   const { worker, socket } = await startPluginWorker();
 
@@ -77,7 +83,7 @@ export async function loadRemoteNxPlugin(
   const pluginPromise = new Promise<LoadedNxPlugin>((res, rej) => {
     sendMessageOverSocket(socket, {
       type: 'load',
-      payload: { plugin, root },
+      payload: { plugin, root, name, pluginPath, shouldRegisterTSTranspiler },
     });
     // logger.verbose(`[plugin-worker] started worker: ${worker.pid}`);
 

packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts

@@ -4,6 +4,7 @@ import { consumeMessagesFromSocket } from '../../../utils/consume-messages-from-
 
 import { createServer } from 'net';
 import { unlinkSync } from 'fs';
+import { registerPluginTSTranspiler } from '../loader';
 
 if (process.env.NX_PERF_LOGGING === 'true') {
   require('../../../utils/perf-logging');
@@ -35,13 +36,30 @@ const server = createServer((socket) => {
         return;
       }
       return consumeMessage(socket, message, {
-        load: async ({ plugin: pluginConfiguration, root }) => {
+        load: async ({
+          plugin: pluginConfiguration,
+          root,
+          name,
+          pluginPath,
+          shouldRegisterTSTranspiler,
+        }) => {
           if (loadTimeout) clearTimeout(loadTimeout);
           process.chdir(root);
           try {
-            const { loadNxPlugin } = await import('../loader');
-            const [promise] = loadNxPlugin(pluginConfiguration, root);
-            plugin = await promise;
+            const { loadResolvedNxPluginAsync } = await import(
+              '../load-resolved-plugin'
+            );
+
+            // Register the ts-transpiler if we are pointing to a
+            // plain ts file that's not part of a plugin project
+            if (shouldRegisterTSTranspiler) {
+              registerPluginTSTranspiler();
+            }
+            plugin = await loadResolvedNxPluginAsync(
+              pluginConfiguration,
+              pluginPath,
+              name
+            );
             return {
               type: 'load-result',
               payload: {

packages/nx/src/project-graph/plugins/load-resolved-plugin.ts

@@ -0,0 +1,26 @@
+import type { PluginConfiguration } from '../../config/nx-json';
+import { LoadedNxPlugin } from './internal-api';
+import { NxPlugin } from './public-api';
+
+export async function loadResolvedNxPluginAsync(
+  pluginConfiguration: PluginConfiguration,
+  pluginPath: string,
+  name: string
+) {
+  const plugin = await importPluginModule(pluginPath);
+  plugin.name ??= name;
+  return new LoadedNxPlugin(plugin, pluginConfiguration);
+}
+
+async function importPluginModule(pluginPath: string): Promise<NxPlugin> {
+  const m = await import(pluginPath);
+  if (
+    m.default &&
+    ('createNodes' in m.default ||
+      'createNodesV2' in m.default ||
+      'createDependencies' in m.default)
+  ) {
+    return m.default;
+  }
+  return m;
+}

packages/nx/src/project-graph/plugins/loader.ts

@@ -24,13 +24,13 @@ import { logger } from '../../utils/logger';
 
 import type * as ts from 'typescript';
 import { extname } from 'node:path';
-import type { NxPlugin } from './public-api';
 import type { PluginConfiguration } from '../../config/nx-json';
 import { retrieveProjectConfigurationsWithoutPluginInference } from '../utils/retrieve-workspace-files';
 import { LoadedNxPlugin } from './internal-api';
 import { LoadPluginError } from '../error-types';
 import path = require('node:path/posix');
 import { readTsConfig } from '../../plugins/js/utils/typescript';
+import { loadResolvedNxPluginAsync } from './load-resolved-plugin';
 
 export function readPluginPackageJson(
   pluginName: string,
@@ -200,18 +200,18 @@ export function getPluginPathAndName(
   root: string
 ) {
   let pluginPath: string;
-  let registerTSTranspiler = false;
+  let shouldRegisterTSTranspiler = false;
   try {
     pluginPath = require.resolve(moduleName, {
       paths,
     });
     const extension = path.extname(pluginPath);
-    registerTSTranspiler = extension === '.ts';
+    shouldRegisterTSTranspiler = extension === '.ts';
   } catch (e) {
     if (e.code === 'MODULE_NOT_FOUND') {
       const plugin = resolveLocalNxPlugin(moduleName, projects, root);
       if (plugin) {
-        registerTSTranspiler = true;
+        shouldRegisterTSTranspiler = true;
         const main = readPluginMainFromProjectConfiguration(
           plugin.projectConfig
         );
@@ -226,18 +226,12 @@ export function getPluginPathAndName(
   }
   const packageJsonPath = path.join(pluginPath, 'package.json');
 
-  // Register the ts-transpiler if we are pointing to a
-  // plain ts file that's not part of a plugin project
-  if (registerTSTranspiler) {
-    registerPluginTSTranspiler();
-  }
-
   const { name } =
     !['.ts', '.js'].some((x) => extname(moduleName) === x) && // Not trying to point to a ts or js file
     existsSync(packageJsonPath) // plugin has a package.json
       ? readJsonFile(packageJsonPath) // read name from package.json
       : { name: moduleName };
-  return { pluginPath, name };
+  return { pluginPath, name, shouldRegisterTSTranspiler };
 }
 
 let projectsWithoutInference: Record<string, ProjectConfiguration>;
@@ -249,6 +243,27 @@ export function loadNxPlugin(plugin: PluginConfiguration, root: string) {
   ] as const;
 }
 
+export async function resolveNxPlugin(
+  moduleName: string,
+  root: string,
+  paths: string[]
+) {
+  try {
+    require.resolve(moduleName);
+  } catch {
+    // If a plugin cannot be resolved, we will need projects to resolve it
+    projectsWithoutInference ??=
+      await retrieveProjectConfigurationsWithoutPluginInference(root);
+  }
+  const { pluginPath, name, shouldRegisterTSTranspiler } = getPluginPathAndName(
+    moduleName,
+    paths,
+    projectsWithoutInference,
+    root
+  );
+  return { pluginPath, name, shouldRegisterTSTranspiler };
+}
+
 export async function loadNxPluginAsync(
   pluginConfiguration: PluginConfiguration,
   paths: string[],
@@ -259,37 +274,14 @@ export async function loadNxPluginAsync(
       ? pluginConfiguration
       : pluginConfiguration.plugin;
   try {
-    try {
-      require.resolve(moduleName);
-    } catch {
-      // If a plugin cannot be resolved, we will need projects to resolve it
-      projectsWithoutInference ??=
-        await retrieveProjectConfigurationsWithoutPluginInference(root);
-    }
-    const { pluginPath, name } = getPluginPathAndName(
-      moduleName,
-      paths,
-      projectsWithoutInference,
-      root
-    );
-    const plugin = await importPluginModule(pluginPath);
-    plugin.name ??= name;
+    const { pluginPath, name, shouldRegisterTSTranspiler } =
+      await resolveNxPlugin(moduleName, root, paths);
 
-    return new LoadedNxPlugin(plugin, pluginConfiguration);
+    if (shouldRegisterTSTranspiler) {
+      registerPluginTSTranspiler();
+    }
+    return loadResolvedNxPluginAsync(pluginConfiguration, pluginPath, name);
   } catch (e) {
     throw new LoadPluginError(moduleName, e);
   }
 }
-
-async function importPluginModule(pluginPath: string): Promise<NxPlugin> {
-  const m = await import(pluginPath);
-  if (
-    m.default &&
-    ('createNodes' in m.default ||
-      'createNodesV2' in m.default ||
-      'createDependencies' in m.default)
-  ) {
-    return m.default;
-  }
-  return m;
-}