product-info: use layout information from product-info.json when buil…

…ding plugin index #SCL-22564 fixed

ideaSupport/src/main/scala/org/jetbrains/sbtidea/Init.scala

@@ -1,13 +1,14 @@
 package org.jetbrains.sbtidea
 
 import org.jetbrains.sbtidea.download.*
+import org.jetbrains.sbtidea.download.plugin.PluginDescriptor
 import org.jetbrains.sbtidea.instrumentation.ManipulateBytecode
 import org.jetbrains.sbtidea.packaging.PackagingKeys.*
 import org.jetbrains.sbtidea.productInfo.ProductInfoParser
 import org.jetbrains.sbtidea.searchableoptions.BuildIndex
 import org.jetbrains.sbtidea.tasks.*
 import sbt.Keys.*
-import sbt.{Attributed, File, file, Keys as _, *}
+import sbt.{Attributed, Def, File, file, Keys as _, *}
 
 import scala.annotation.nowarn
 import scala.collection.mutable
@@ -83,47 +84,56 @@ trait Init { this: Keys.type =>
     }) compose (onLoad in Global).value
   )
 
+  private def pluginsClasspath: Def.Initialize[Task[Seq[(PluginDescriptor, Classpath)]]] = Def.taskDyn {
+    getPluginsClasspath(intellijPlugins.value)
+  }
+
+  private def runtimePluginsClasspath: Def.Initialize[Task[Seq[(PluginDescriptor, Classpath)]]] = Def.taskDyn {
+    getPluginsClasspath(intellijRuntimePlugins.value)
+  }
+
+  private def getPluginsClasspath(plugins: Seq[IntellijPlugin]): Def.Initialize[Task[Seq[(PluginDescriptor, Classpath)]]] = Def.task {
+    tasks.CreatePluginsClasspath.buildPluginClassPaths(
+      intellijBaseDirectory.in(ThisBuild).value.toPath,
+      BuildInfo(
+        intellijBuild.in(ThisBuild).value,
+        intellijPlatform.in(ThisBuild).value
+      ),
+      plugins,
+      new SbtPluginLogger(streams.value),
+      intellijAttachSources.in(Global).value,
+      name.value
+    )
+  }
+
   lazy val projectSettings: Seq[Setting[?]] = Seq(
     intellijMainJars := {
       val intellijBaseDir = Keys.intellijBaseDirectory.in(ThisBuild).value
       val productInfo = Keys.productInfo.in(ThisBuild).value
       val jbrPlatform = jbrInfo.in(ThisBuild).value.platform
 
       val bootstrapJars = productInfo.bootClasspathJars(jbrPlatform, intellijBaseDir)
-      val coreModulesJars = productInfo.coreModulesJars(intellijBaseDir)
-      Attributed.blankSeq(bootstrapJars ++ coreModulesJars)
+      val productModulesJars = productInfo.productModulesJars(intellijBaseDir)
+      Attributed.blankSeq(bootstrapJars ++ productModulesJars)
+    },
+    intellijTestJars := {
+      val intellijBaseDir = Keys.intellijBaseDirectory.in(ThisBuild).value
+      val productInfo = Keys.productInfo.in(ThisBuild).value
+      Attributed.blankSeq(productInfo.testFrameworkJars(intellijBaseDir))
     },
     intellijPlugins := Seq.empty,
     intellijRuntimePlugins := Seq.empty,
-    intellijPluginJars :=
-      tasks.CreatePluginsClasspath.buildPluginClassPaths(
-        intellijBaseDirectory.in(ThisBuild).value.toPath,
-        BuildInfo(
-          intellijBuild.in(ThisBuild).value,
-          intellijPlatform.in(ThisBuild).value
-        ),
-        intellijPlugins.value,
-        new SbtPluginLogger(streams.value),
-        intellijAttachSources.in(Global).value,
-        name.value),
+    intellijPluginJars := pluginsClasspath.value,
 
-    externalDependencyClasspath in Compile ++= UpdateWithIDEAInjectionTask.buildExternalDependencyClassPath.value,
+    Compile / externalDependencyClasspath ++= UpdateWithIDEAInjectionTask.buildExternalDependencyClassPath.value,
     Runtime / externalDependencyClasspath  ++= {
       val cp = (Compile / externalDependencyClasspath).value
-      val runtimePlugins = tasks.CreatePluginsClasspath.buildPluginClassPaths(
-        intellijBaseDirectory.in(ThisBuild).value.toPath,
-        BuildInfo(
-          intellijBuild.in(ThisBuild).value,
-          intellijPlatform.in(ThisBuild).value
-        ),
-        intellijRuntimePlugins.value,
-        new SbtPluginLogger(streams.value),
-        intellijAttachSources.in(Global).value,
-        name.value
-      ).flatMap(_._2)
+      val runtimePlugins = runtimePluginsClasspath.value.flatMap(_._2)
       cp ++ runtimePlugins
     },
-    externalDependencyClasspath in Test    ++= (externalDependencyClasspath in Runtime).value,
+    Test / externalDependencyClasspath ++= {
+      UpdateWithIDEAInjectionTask.buildTestExternalDependencyClassPath.value ++ (Runtime / externalDependencyClasspath).value
+    },
 
     update := UpdateWithIDEAInjectionTask.createTask.value,
 

ideaSupport/src/main/scala/org/jetbrains/sbtidea/Keys.scala

@@ -58,6 +58,9 @@ object Keys extends Defns with Init with Utils with Quirks {
   lazy val intellijMainJars = taskKey[Classpath](
     "Classpath containing main IntelliJ Platform jars")
 
+  lazy val intellijTestJars = taskKey[Classpath](
+    "Classpath containing IntelliJ Platform test framework jars")
+
   lazy val productInfo = taskKey[ProductInfo](
     "Information about IntelliJ distribution extracted from product-info.json file from IntelliJ Platform root directory")
 

ideaSupport/src/main/scala/org/jetbrains/sbtidea/download/CommunityUpdater.scala

@@ -18,13 +18,16 @@ class CommunityUpdater(
 ) {
 
   implicit protected val context: InstallContext =
-    InstallContext(baseDirectory = baseDirectory, downloadDirectory = baseDirectory.getParent)
+    InstallContext(
+      baseDirectory = baseDirectory,
+      downloadDirectory = baseDirectory.getParent,
+    )
 
   implicit protected val remoteRepoApi: PluginRepoUtils =
     new PluginRepoUtils
 
   implicit protected val localRegistry: LocalPluginRegistry =
-    new LocalPluginRegistry(baseDirectory)
+    new LocalPluginRegistry(context)
 
   protected val ideaDependency: IdeaDependency = IdeaDependency(ideaBuildInfo)
 

ideaSupport/src/main/scala/org/jetbrains/sbtidea/download/api/InstallContext.scala

@@ -1,5 +1,13 @@
 package org.jetbrains.sbtidea.download.api
 
+import org.jetbrains.sbtidea.productInfo.{ProductInfo, ProductInfoParser}
+import sbt.pathToPathOps
+
 import java.nio.file.Path
 
-case class InstallContext(baseDirectory: Path, downloadDirectory: Path)
+case class InstallContext(baseDirectory: Path, downloadDirectory: Path) {
+  lazy val productInfo: ProductInfo = {
+    val productInfoFile = baseDirectory / "product-info.json"
+    ProductInfoParser.parse(productInfoFile.toFile)
+  }
+}

ideaSupport/src/main/scala/org/jetbrains/sbtidea/download/plugin/LocalPluginRegistry.scala

@@ -1,16 +1,18 @@
 package org.jetbrains.sbtidea.download.plugin
 
 import org.jetbrains.sbtidea.download.*
+import org.jetbrains.sbtidea.download.api.InstallContext
 import org.jetbrains.sbtidea.{IntellijPlugin, PathExt, PluginLogger as log}
 import sbt.*
 
 import java.nio.file.{Files, Path}
-import scala.collection.mutable
 
-class LocalPluginRegistry (ideaRoot: Path) extends LocalPluginRegistryApi {
+class LocalPluginRegistry(ctx: InstallContext) extends LocalPluginRegistryApi {
   import LocalPluginRegistry.*
 
-  val index = new PluginIndexImpl(ideaRoot)
+  private val ideaRoot = ctx.baseDirectory
+
+  lazy val index = new PluginIndexImpl(ideaRoot)
 
   private def getDescriptorFromPluginFolder(name: String): Either[String, PluginDescriptor] =
     extractPluginMetaData(ideaRoot / "plugins" / name)
@@ -83,9 +85,6 @@ class LocalPluginRegistry (ideaRoot: Path) extends LocalPluginRegistryApi {
 
 object LocalPluginRegistry {
 
-  private val instances: mutable.Map[Path, LocalPluginRegistry] =
-    new mutable.WeakHashMap[Path, LocalPluginRegistry]().withDefault(new LocalPluginRegistry(_))
-
   private class MissingPluginRootException(pluginName: String)
     extends RuntimeException(s"Can't find plugin root for $pluginName: check plugin name")
 
@@ -130,6 +129,4 @@ object LocalPluginRegistry {
     }
     descriptorFinal
   }
-
-  def instanceFor(ideaRoot: Path): LocalPluginRegistry = instances(ideaRoot)
 }

ideaSupport/src/main/scala/org/jetbrains/sbtidea/download/plugin/PluginDescriptor.scala

@@ -47,7 +47,7 @@ case class PluginDescriptor(id: String,
    *
    * Most of the plugins keep all information in single file `plugin.xml` (id, name, sinceBuild, etc...)<br>
    * However some plugins can keep this information in multiple files.
-   * For example "Code With Me" has different implementations for IDEA and Rider.
+   * For example, "Code With Me" has different implementations for IDEA and Rider.
    * They keep common parts in `pluginBase.xml` (like id, name, vendor) and other parts (IDE-specific) in `plugin.xml` file.
    *
    * Alternative approach could be to truely resolve all "include" directives in `plugin.xml`, like: {{{
@@ -74,22 +74,6 @@ case class PluginDescriptor(id: String,
 }
 
 object PluginDescriptor {
-  /**
-   * See [[PluginIndexImpl.buildFromProductModulesDir]].
-   *
-   * We cannot parse a descriptor from module's plugin.xml because it doesn't have one.
-   * The only xml file modules have (and it is located in jar's root, i.e.: outside META-INF directory)
-   * will not have any of the fields below.
-   *
-   * Minimal example of a module without extensions, actions and even optional package:
-   * {{{
-   *   <idea-plugin>
-   *   </idea-plugin>
-   * }}}
-   */
-  private[sbtidea] def apply(id: String): PluginDescriptor =
-    new PluginDescriptor(id = id, vendor = "", name = "", version = "", sinceBuild = "", untilBuild = "")
-
   private val OPTIONAL_KEY  = "(optional) "
   private val OPTIONAL_ATTR = "optional"
 

ideaSupport/src/main/scala/org/jetbrains/sbtidea/download/plugin/PluginIndexImpl.scala

@@ -2,6 +2,7 @@ package org.jetbrains.sbtidea.download.plugin
 
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.sbtidea.download.plugin.LocalPluginRegistry.extractPluginMetaData
+import org.jetbrains.sbtidea.productInfo.ProductInfo
 import org.jetbrains.sbtidea.{PathExt, PluginLogger as log}
 import sbt.*
 
@@ -45,22 +46,20 @@ class PluginIndexImpl(ideaRoot: Path) extends PluginIndex {
   }
 
   private def buildAndSaveIndex(): Repr = {
-    val plugins = buildFromPluginsDir()
-    val productModules = buildFromProductModulesDir()
-    val allEntries = plugins ++ productModules
+    val plugins = buildFromPluginsDir
     try {
-      val pluginIds = allEntries.keys.toSeq
+      val pluginIds = plugins.keys.toSeq
         .filter(_.trim.nonEmpty) // for some reason, there is some empty id
         .sorted
         .mkString(", ")
       log.info(s"Plugin ids from $INDEX_FILENAME: $pluginIds")
 
-      saveToFile(allEntries)
+      saveToFile(plugins)
     } catch {
       case e: Throwable =>
         log.warn(s"Failed to write back plugin index: $e")
     }
-    allEntries
+    plugins
   }
 
   override def put(descriptor: PluginDescriptor, installPath: Path): Unit = {
@@ -115,7 +114,7 @@ class PluginIndexImpl(ideaRoot: Path) extends PluginIndex {
     }
   }
 
-  private def buildFromPluginsDir(): Map[PluginId, (Path, PluginDescriptor)] = {
+  private def buildFromPluginsDir: Map[PluginId, (Path, PluginDescriptor)] = {
     val pluginDirs = Files.list(ideaRoot.resolve("plugins")).collect(Collectors.toList[Path]).asScala
     pluginDirs.flatMap { pluginDir =>
       val pluginMetaData = extractPluginMetaData(pluginDir)
@@ -128,22 +127,6 @@ class PluginIndexImpl(ideaRoot: Path) extends PluginIndex {
       }
     }.toMap
   }
-
-  /**
-   * See SCL-22564
-   *
-   * This is a workaround to avoid sbt import failures on productModuleV2 dependency resolution.
-   * V2 modules should be taken from the `layout` field of `product-info.json`.
-   */
-  private def buildFromProductModulesDir(): Map[PluginId, (Path, PluginDescriptor)] = {
-    val modulesDir = ideaRoot.resolve("lib").resolve("modules").toFile
-    if (modulesDir.isDirectory) {
-      modulesDir.listFiles((_, name) => name.endsWith(".jar")).map { file =>
-        val moduleName = file.getName.stripSuffix(".jar")
-        moduleName -> (file.toPath, PluginDescriptor(moduleName))
-      }.toMap
-    } else Map.empty
-  }
 }
 
 object PluginIndexImpl {

ideaSupport/src/main/scala/org/jetbrains/sbtidea/download/plugin/PluginResolver.scala

@@ -12,11 +12,8 @@ class PluginResolver(
   private val resolveSettings: IntellijPlugin.Settings
 )(implicit ctx: InstallContext, repo: PluginRepoApi, localRegistry: LocalPluginRegistryApi) extends Resolver[PluginDependency] {
 
-  // modules that are inside idea.jar
-  private val INTERNAL_MODULE_PREFIX = "com.intellij.modules."
-
   //Keeping multiple errors in `Seq[String]` mainly for the case when IntellijPlugin.Id.fallbackDownloadUrl is not empty
-  //In that case we try to resolve the artifact twice and want to report both errors
+  //In that case, we try to resolve the artifact twice and want to report both errors
   private type PluginDescriptorAndArtifactResolveResult = Either[Seq[String], (PluginDescriptor, PluginArtifact)]
 
   override def resolve(pluginDependency: PluginDependency): Seq[PluginArtifact] = {
@@ -51,10 +48,12 @@ class PluginResolver(
   }
 
   private def resolveDependencies(plugin: PluginDependency, key: IntellijPlugin, descriptor: PluginDescriptor): Seq[PluginArtifact] = {
+    val productInfo = ctx.productInfo
+    val coreModules = productInfo.modules ++ productInfo.productModulesNames
     val dependencies = descriptor.dependsOn
       .filterNot(!resolveSettings.optionalDeps && _.optional)                              // skip all optional plugins if flag is set
-      .filterNot(dep => resolveSettings.excludedIds.contains(dep.id))                      // remove plugins specified by user blacklist
-      .filterNot(_.id.startsWith(INTERNAL_MODULE_PREFIX))                                  // skip plugins embedded in idea.jar
+      .filterNot(dep => resolveSettings.excludedIds.contains(dep.id))                      // remove plugins specified by user blocklist
+      .filterNot(dep => coreModules.contains(dep.id))                                       // skip dependencies on core product modules
       .filterNot(dep => dep.optional && !localRegistry.isPluginInstalled(dep.id.toPlugin)) // skip optional non-bundled plugins
 
     dependencies

ideaSupport/src/main/scala/org/jetbrains/sbtidea/productInfo/ProductInfo.scala

@@ -15,7 +15,12 @@ import java.io.File
  * See also [[org.jetbrains.sbtidea.download.BuildInfo]]
  */
 case class ProductInfo(
+  name: String,
+  version: String,
+  versionSuffix: String,
   buildNumber: String,
+  productCode: String,
+  modules: Seq[String],
   launch: Seq[Launch],
   layout: Seq[LayoutItem]
 ) {
@@ -25,11 +30,20 @@ case class ProductInfo(
     launch.bootClassPathJarNames.map(jarName => intellijBaseDir / "lib" / jarName)
   }
 
-  def coreModulesJars(intellijBaseDir: File): Seq[File] =
-    layout.filter(_.kind == LayoutItemKind.ProductModuleV2)
+  def testFrameworkJars(intellijBaseDir: File): Seq[File] =
+    Seq(intellijBaseDir / "lib" / "testFramework.jar")
+
+  def productModulesNames: Seq[String] =
+    productModulesLayout.map(_.name)
+
+  def productModulesJars(intellijBaseDir: File): Seq[File] =
+    productModulesLayout
       .flatMap(_.classPath.toSeq.flatten)
       .map(intellijBaseDir / _)
 
+  private def productModulesLayout: Seq[LayoutItem] =
+    layout.filter(_.kind == LayoutItemKind.ProductModuleV2)
+
   /**
    * Finds the [[Launch]] object for the given OS and architecture corresponding to the [[JbrPlatform]]
    */
@@ -75,6 +89,7 @@ object LayoutItemKind {
   case object PluginAlias extends LayoutItemKind
   case object ProductModuleV2 extends LayoutItemKind
   case object ModuleV2 extends LayoutItemKind
+  case class Unknown(value: String) extends LayoutItemKind
 }
 
 /**

ideaSupport/src/main/scala/org/jetbrains/sbtidea/productInfo/ProductInfoParser.scala

@@ -1,10 +1,10 @@
 package org.jetbrains.sbtidea.productInfo
 
+import org.jetbrains.sbtidea.PluginLogger as log
 import spray.json.DefaultJsonProtocol.*
 
 import java.io.File
 
-//TODO: unify with org.jetbrains.sbtidea.download.BuildInfoOps#getActualIdeaBuild
 object ProductInfoParser {
 
   import spray.json.*
@@ -19,7 +19,7 @@ object ProductInfoParser {
     jsonAst.convertTo[ProductInfo]
   }
 
-  private implicit def productInfoFormat: RootJsonFormat[ProductInfo] = jsonFormat3(ProductInfo)
+  private implicit def productInfoFormat: RootJsonFormat[ProductInfo] = jsonFormat8(ProductInfo)
   private implicit def launchFormat: JsonFormat[Launch] = jsonFormat8(Launch)
   private implicit def layoutItemFormat: RootJsonFormat[LayoutItem] = jsonFormat3(LayoutItem)
 
@@ -30,8 +30,10 @@ object ProductInfoParser {
         case "plugin" => LayoutItemKind.Plugin
         case "pluginAlias" => LayoutItemKind.PluginAlias
         case "productModuleV2" => LayoutItemKind.ProductModuleV2
-        case unknown =>
-          throw new RuntimeException(s"Unknown layout item kind: $unknown")
+        case value =>
+          log.warn(s"Unknown layout item kind: $value")
+          //use special "Unknown" case class to be more fail-tolerant
+          LayoutItemKind.Unknown(value)
       }
       case _ =>
         deserializationError("LayoutItemKind expected")
@@ -47,6 +49,8 @@ object ProductInfoParser {
         case "windows" => OS.Windows
         case "macos" => OS.macOs
         case "linux" => OS.Linux
+        case _ =>
+          throw new RuntimeException(s"Unknown OS: $value")
       }
       case _ =>
         deserializationError("OS expected")