From 64b51a4e0d9e371eb6730afeda386aabed49c405 Mon Sep 17 00:00:00 2001 From: Jeremiah Zucker Date: Wed, 10 Jul 2024 21:34:48 -0400 Subject: [PATCH] revamped jmh perf --- jvm/hermes/BUILD | 5 +- .../hermes/bridge/runtime/HermesRuntime.kt | 4 +- .../playerui/j2v8/bridge/runtime/V8Runtime.kt | 3 +- jvm/perf/BUILD | 7 +- .../intuit/playerui/perf/jmh/PlayerPerf.kt | 260 ++++++------------ 5 files changed, 100 insertions(+), 179 deletions(-) diff --git a/jvm/hermes/BUILD b/jvm/hermes/BUILD index 2dbe5ba74..840571990 100644 --- a/jvm/hermes/BUILD +++ b/jvm/hermes/BUILD @@ -17,7 +17,10 @@ kt_jvm_library( name = "hermes-host", associates = [":hermes"], visibility = ["//visibility:public"], - exports = ["//jvm/hermes/src/main/jni:resources"], + exports = [ + ":hermes", + "//jvm/hermes/src/main/jni:resources", + ], ) android_library( diff --git a/jvm/hermes/src/main/kotlin/com/intuit/playerui/hermes/bridge/runtime/HermesRuntime.kt b/jvm/hermes/src/main/kotlin/com/intuit/playerui/hermes/bridge/runtime/HermesRuntime.kt index e79d4e880..99fa1d56b 100644 --- a/jvm/hermes/src/main/kotlin/com/intuit/playerui/hermes/bridge/runtime/HermesRuntime.kt +++ b/jvm/hermes/src/main/kotlin/com/intuit/playerui/hermes/bridge/runtime/HermesRuntime.kt @@ -203,7 +203,9 @@ public object Hermes : PlayerRuntimeFactory { return HermesRuntime.create(config).also(SetTimeoutPlugin(config.coroutineExceptionHandler)::apply) } - override fun toString(): String = "Hermes" + override fun toString(): String = name + + public const val name: String = "Hermes" } public class HermesRuntimeContainer : PlayerRuntimeContainer { diff --git a/jvm/j2v8/src/main/kotlin/com/intuit/playerui/j2v8/bridge/runtime/V8Runtime.kt b/jvm/j2v8/src/main/kotlin/com/intuit/playerui/j2v8/bridge/runtime/V8Runtime.kt index 2cc3b437a..255de3e59 100644 --- a/jvm/j2v8/src/main/kotlin/com/intuit/playerui/j2v8/bridge/runtime/V8Runtime.kt +++ b/jvm/j2v8/src/main/kotlin/com/intuit/playerui/j2v8/bridge/runtime/V8Runtime.kt @@ -205,7 +205,8 @@ public object J2V8 : PlayerRuntimeFactory { override fun create(block: J2V8RuntimeConfig.() -> Unit): Runtime = V8Runtime(J2V8RuntimeConfig().apply(block)) - override fun toString(): String = "J2V8" + override fun toString(): String = name + public const val name: String = "J2V8" } public data class J2V8RuntimeConfig( diff --git a/jvm/perf/BUILD b/jvm/perf/BUILD index e36b7e9c9..325201876 100644 --- a/jvm/perf/BUILD +++ b/jvm/perf/BUILD @@ -1,5 +1,5 @@ -load("//jvm:defs.bzl", "kt_player_module") load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") +load("//jvm:defs.bzl", "kt_player_module") kt_player_module( name = "perf_tests", @@ -21,15 +21,16 @@ kt_jvm_library( plugins = [":jmh_annotation_processor"], resources = glob(["src/main/resources/*.json"]), deps = [ - "//jvm:kotlin_serialization", "//jvm/core", "//jvm/graaljs", + "//jvm/hermes:hermes-host", "//jvm/j2v8:j2v8-all", - "@rules_kotlin//kotlin/compiler:kotlin-reflect", + "//tools/mocks:jar", "@maven//:org_openjdk_jmh_jmh_core", ], ) +# TODO: Potentially transition to ensure //:cmake_build_type is MinSizeRel java_binary( name = "perf", main_class = "org.openjdk.jmh.Main", diff --git a/jvm/perf/src/main/kotlin/com/intuit/playerui/perf/jmh/PlayerPerf.kt b/jvm/perf/src/main/kotlin/com/intuit/playerui/perf/jmh/PlayerPerf.kt index d3fdb0927..f5144a6cb 100644 --- a/jvm/perf/src/main/kotlin/com/intuit/playerui/perf/jmh/PlayerPerf.kt +++ b/jvm/perf/src/main/kotlin/com/intuit/playerui/perf/jmh/PlayerPerf.kt @@ -1,21 +1,19 @@ package com.intuit.playerui.perf.jmh -import com.intuit.playerui.core.asset.Asset -import com.intuit.playerui.core.bridge.Invokable -import com.intuit.playerui.core.bridge.Node -import com.intuit.playerui.core.bridge.Promise +import com.intuit.playerui.core.bridge.runtime.PlayerRuntimeConfig import com.intuit.playerui.core.bridge.runtime.PlayerRuntimeFactory import com.intuit.playerui.core.bridge.runtime.Runtime import com.intuit.playerui.core.bridge.runtime.runtimeContainers -import com.intuit.playerui.core.data.DataController -import com.intuit.playerui.core.data.set import com.intuit.playerui.core.player.HeadlessPlayer +import com.intuit.playerui.hermes.bridge.runtime.Hermes +import com.intuit.playerui.j2v8.bridge.runtime.J2V8 +import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking import kotlinx.serialization.Contextual -import kotlinx.serialization.Serializable import org.openjdk.jmh.annotations.Benchmark import org.openjdk.jmh.annotations.BenchmarkMode import org.openjdk.jmh.annotations.CompilerControl +import org.openjdk.jmh.annotations.Level.Invocation import org.openjdk.jmh.annotations.Mode import org.openjdk.jmh.annotations.OutputTimeUnit import org.openjdk.jmh.annotations.Param @@ -27,6 +25,7 @@ import java.util.concurrent.TimeUnit import kotlin.coroutines.resume import kotlin.coroutines.suspendCoroutine + @Serializable data class Update( val binding: String, @@ -53,7 +52,11 @@ val mocks: Map = mapOf( ContentID("three") to Update( "foo", 3 - ) + ), + ContentID("mocks/info/info-modal-flow") to Update( + "foo", + 3 + ), ) private const val playerSourcePath = "core/player/dist/Player.native.js" @@ -72,208 +75,119 @@ private fun readResource(path: String): String = classLoader.getResource(path)?. @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) -@State(Scope.Benchmark) -abstract class RuntimePerformance { +@State(Scope.Thread) +public abstract class RuntimePerformance { + + @Param(Hermes.name, J2V8.name) + protected lateinit var runtimeName: String; private set - @Param("J2V8", "Graal") - lateinit var runtime: String + private lateinit var factory: PlayerRuntimeFactory<*> - lateinit var factory: PlayerRuntimeFactory<*> + protected lateinit var runtime: Runtime<*>; private set - protected fun findRuntimeFactory(runtime: String): PlayerRuntimeFactory<*> = - runtimeContainers.single { "$it" == runtime }.factory + @Setup public fun setupRuntimeFactory() { + factory = findRuntimeFactory(runtimeName) + } - protected fun setupRuntime(runtime: String = this.runtime) { - factory = findRuntimeFactory(runtime) + protected fun setupRuntime(block: PlayerRuntimeConfig.() -> Unit = {}): Runtime<*> = factory.create(block).also { + runtime = it } + + private fun findRuntimeFactory(runtimeName: String): PlayerRuntimeFactory<*> = + runtimeContainers.single { "$it" == runtimeName }.factory } -open class RuntimeCreation : RuntimePerformance() { +public open class BenchRuntimeCreation : RuntimePerformance() { - @Setup fun setup() { - setupRuntime() + @CompilerControl(CompilerControl.Mode.DONT_INLINE) + @Benchmark public fun createRuntime(consumer: Blackhole) { + // TODO: Somehow this creates a segfault - this only happens very rarely + // but it's really weird. At least it's happening in such a simple + // test. Something might not being tracked when creating the runtime? + // UPDATE: Ohhh, we're attaching the SetTimeoutPlugin. Which registers + // host functions. Great, we'll need to make sure we're guarding those + // appropriately. + // UPDATE 2: Well, the segfault happens on objects, which won't be + // UPDATE 3: Only global & empty are created :( so, tracking ref isn't foolproof? + // UPDATE 4: Reproduced it with value, so now i'm thinking tracking isn't foolproof + // UPDATE 5: Weak ref is killing us :( + consumer.consume(setupRuntime()) + runtime.release() } @CompilerControl(CompilerControl.Mode.DONT_INLINE) - @Benchmark fun createRuntime(consumer: Blackhole) { - consumer.consume(factory.create()) + @Benchmark fun createHeadlessPlayer(consumer: Blackhole) { + // measures runtime headless player creation + runtime creation + // this doesn't technically work the same way it does when the + // player usually is created, because it just pulls the first + // runtime factory it can find. we use our setupRuntime method + // to ensure we release the runtime. + consumer.consume(HeadlessPlayer(explicitRuntime = setupRuntime())) + // we always release the runtime to make sure jmh isn't waiting + // for the runtime thread to be released + runtime.release() } - } -open class RuntimePlayerCreation : RuntimePerformance() { - - lateinit var jsRuntime: Runtime<*> +public open class BenchPlayerCreation : RuntimePerformance() { - @Setup fun setup() { - setupRuntime(runtime) - jsRuntime = factory.create() + // if we don't release the runtime on each bench, we wouldn't need to reload the runtime + // on each invocation it might be useful to see how "warm" the runtime can actually get + @Setup(Invocation) public fun setup() { + // load up runtime and player source in setup to isolate benchmarks + setupRuntime() playerSource } @CompilerControl(CompilerControl.Mode.DONT_INLINE) - @Benchmark fun createHeadlessPlayer(consumer: Blackhole) { - consumer.consume(HeadlessPlayer(explicitRuntime = jsRuntime)) + @Benchmark public fun createJSPlayer(consumer: Blackhole) { + consumer.consume(runtime.execute(playerSource)) + consumer.consume(runtime.execute("""(new Player.Player())""")) + runtime.release() } @CompilerControl(CompilerControl.Mode.DONT_INLINE) - @Benchmark fun createPlayer(consumer: Blackhole) { - jsRuntime.execute(playerSource) - consumer.consume(jsRuntime.execute("""(new Player.Player())""")) + @Benchmark public fun createHeadlessPlayer(consumer: Blackhole) { + // uses runtime created outside benchmark + consumer.consume(HeadlessPlayer(explicitRuntime = runtime)) + runtime.release() } - } -open class ContentPerformance : RuntimePerformance() { - - @Param("w2", "pi", "cli") - lateinit var content: String - - lateinit var flow: String - - lateinit var update: Update - - lateinit var jsRuntime: Runtime<*> - - protected fun readContent(content: String): String = - readResource("$content.json") +public open class BenchPlayerFlow : RuntimePerformance() { - protected fun setupContent(content: String = this.content) { - setupRuntime() - jsRuntime = factory.create() - flow = readContent(content) - update = mocks[ContentID(content)]!! - } - -} + @Param("mocks/info/info-modal-flow") + public lateinit var content: String -open class HeadlessPlayerFlowPerformance : ContentPerformance() { + public lateinit var flow: String - lateinit var runtimeFactory: PlayerRuntimeFactory<*> + public lateinit var update: Update - @Setup fun setup() { - setupContent() + @Setup(Invocation) + public fun setup() { + flow = readResource("$content.json") + update = mocks[ContentID(content)] ?: throw RuntimeException("update not defined for $content") } @CompilerControl(CompilerControl.Mode.DONT_INLINE) - @Benchmark fun startFlow(consumer: Blackhole) { - val player = HeadlessPlayer(explicitRuntime = jsRuntime) - consumer.consume(runBlocking { - suspendCoroutine { - player.hooks.viewController.tap("perf") { vc -> - vc?.hooks?.view?.tap("new view") { v -> - v?.hooks?.onUpdate?.tap("test") { a -> + @Benchmark + fun firstViewUpdate(consumer: Blackhole) { + val player = HeadlessPlayer(explicitRuntime = setupRuntime()) + val pending = player.scope.async { + suspendCoroutine { + player.hooks.viewController.tap { vc -> + vc?.hooks?.view?.tap { v -> + v?.hooks?.onUpdate?.tap { a -> it.resume(a) } } } - - player.start(flow) - } - }) - } - - @CompilerControl(CompilerControl.Mode.DONT_INLINE) - @Benchmark fun startAndUpdateFlow(consumer: Blackhole) { - val player = HeadlessPlayer(explicitRuntime = jsRuntime) - consumer.consume(runBlocking { - suspendCoroutine { - var dataController: DataController? = null - player.hooks.dataController.tap("perf") { dc -> - dataController = dc - } - - var updateCount = 0 - player.hooks.viewController.tap("perf") { vc -> - vc?.hooks?.view?.tap("new view") { v -> - v?.hooks?.onUpdate?.tap("test") { a -> - if (updateCount++ == 0) - dataController!!.set(update.binding to update.value) - - else it.resume(a) - } - } - } - player.start(flow) } - }) - } - -} - -open class RuntimePlayerFlowPerformance : ContentPerformance() { - - lateinit var runtimeFactory: PlayerRuntimeFactory<*> - - lateinit var flowNode: Node - - @Setup fun setup() { - setupContent() - flowNode = jsRuntime.execute("""($flow)""") as Node - } - - @CompilerControl(CompilerControl.Mode.DONT_INLINE) - @Benchmark fun startFlow(consumer: Blackhole) { - jsRuntime.execute(playerSource) - - val runner = jsRuntime.execute(""" -((content, playerFactory = () => new Player.Player()) => { - return new Promise((resolve, reject) => { - try { - const player = playerFactory(); - player.hooks.viewController.tap("perf", (vc) => { - vc.hooks.view.tap("new view", (v) => { - v.hooks.onUpdate.tap("test", (a) => { - resolve(a); - }); - }); - }); - player.start(content).catch(reject); - } catch (e) { reject(e); } - }); -}) -""") as Invokable - val promise = Promise(runner(flowNode)) - - consumer.consume(runBlocking { - promise.toCompletable(Asset.serializer()).await() - }) + } + runBlocking { + pending.await() + } + runtime.release() } - - @CompilerControl(CompilerControl.Mode.DONT_INLINE) - @Benchmark fun startAndUpdateFlow(consumer: Blackhole) { - jsRuntime.execute(playerSource) - - val runner = jsRuntime.execute(""" -((content, update, playerFactory = () => new Player.Player()) => { - return new Promise((resolve, reject) => { - try { - const player = playerFactory(); - let dataController; - player.hooks.dataController.tap("perf", (dc) => { - dataController = dc; - }); - let updateCount = 0; - player.hooks.viewController.tap("perf", (vc) => { - vc.hooks.view.tap("new view", (v) => { - v.hooks.onUpdate.tap("test", (a) => { - if (updateCount++ == 0) - dataController.set({ [update.binding]: update.value }); - else resolve(a); - }); - }); - }); - player.start(content).catch(reject); - } catch (e) { reject(e); } - }); -}) -""") as Invokable - val promise = Promise(runner(flowNode, update)) - - consumer.consume(runBlocking { - promise.toCompletable(Asset.serializer()).await() - }) - } - }