revamped jmh perf

jvm/hermes/BUILD

@@ -17,7 +17,10 @@ kt_jvm_library(
     name = "hermes-host",
     associates = [":hermes"],
     visibility = ["//visibility:public"],
-    exports = ["//jvm/hermes/src/main/jni:resources"],
+    exports = [
+        ":hermes",
+        "//jvm/hermes/src/main/jni:resources",
+    ],
 )
 
 android_library(

jvm/hermes/src/main/kotlin/com/intuit/playerui/hermes/bridge/runtime/HermesRuntime.kt

@@ -203,7 +203,9 @@ public object Hermes : PlayerRuntimeFactory<Config> {
         return HermesRuntime.create(config).also(SetTimeoutPlugin(config.coroutineExceptionHandler)::apply)
     }
 
-    override fun toString(): String = "Hermes"
+    override fun toString(): String = name
+
+    public const val name: String = "Hermes"
 }
 
 public class HermesRuntimeContainer : PlayerRuntimeContainer {

jvm/j2v8/src/main/kotlin/com/intuit/playerui/j2v8/bridge/runtime/V8Runtime.kt

@@ -205,7 +205,8 @@ public object J2V8 : PlayerRuntimeFactory<J2V8RuntimeConfig> {
     override fun create(block: J2V8RuntimeConfig.() -> Unit): Runtime<V8Value> =
         V8Runtime(J2V8RuntimeConfig().apply(block))
 
-    override fun toString(): String = "J2V8"
+    override fun toString(): String = name
+    public const val name: String = "J2V8"
 }
 
 public data class J2V8RuntimeConfig(

jvm/perf/BUILD

@@ -1,5 +1,5 @@
-load("//jvm:defs.bzl", "kt_player_module")
 load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
+load("//jvm:defs.bzl", "kt_player_module")
 
 kt_player_module(
     name = "perf_tests",
@@ -21,15 +21,16 @@ kt_jvm_library(
     plugins = [":jmh_annotation_processor"],
     resources = glob(["src/main/resources/*.json"]),
     deps = [
-        "//jvm:kotlin_serialization",
         "//jvm/core",
         "//jvm/graaljs",
+        "//jvm/hermes:hermes-host",
         "//jvm/j2v8:j2v8-all",
-        "@rules_kotlin//kotlin/compiler:kotlin-reflect",
+        "//tools/mocks:jar",
         "@maven//:org_openjdk_jmh_jmh_core",
     ],
 )
 
+# TODO: Potentially transition to ensure //:cmake_build_type is MinSizeRel
 java_binary(
     name = "perf",
     main_class = "org.openjdk.jmh.Main",

jvm/perf/src/main/kotlin/com/intuit/playerui/perf/jmh/PlayerPerf.kt

@@ -1,21 +1,19 @@
 package com.intuit.playerui.perf.jmh
 
-import com.intuit.playerui.core.asset.Asset
-import com.intuit.playerui.core.bridge.Invokable
-import com.intuit.playerui.core.bridge.Node
-import com.intuit.playerui.core.bridge.Promise
+import com.intuit.playerui.core.bridge.runtime.PlayerRuntimeConfig
 import com.intuit.playerui.core.bridge.runtime.PlayerRuntimeFactory
 import com.intuit.playerui.core.bridge.runtime.Runtime
 import com.intuit.playerui.core.bridge.runtime.runtimeContainers
-import com.intuit.playerui.core.data.DataController
-import com.intuit.playerui.core.data.set
 import com.intuit.playerui.core.player.HeadlessPlayer
+import com.intuit.playerui.hermes.bridge.runtime.Hermes
+import com.intuit.playerui.j2v8.bridge.runtime.J2V8
+import kotlinx.coroutines.async
 import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.Contextual
-import kotlinx.serialization.Serializable
 import org.openjdk.jmh.annotations.Benchmark
 import org.openjdk.jmh.annotations.BenchmarkMode
 import org.openjdk.jmh.annotations.CompilerControl
+import org.openjdk.jmh.annotations.Level.Invocation
 import org.openjdk.jmh.annotations.Mode
 import org.openjdk.jmh.annotations.OutputTimeUnit
 import org.openjdk.jmh.annotations.Param
@@ -27,6 +25,7 @@ import java.util.concurrent.TimeUnit
 import kotlin.coroutines.resume
 import kotlin.coroutines.suspendCoroutine
 
+
 @Serializable
 data class Update(
     val binding: String,
@@ -53,7 +52,11 @@ val mocks: Map<ContentID, Update> = mapOf(
     ContentID("three") to Update(
         "foo",
         3
-    )
+    ),
+    ContentID("mocks/info/info-modal-flow") to Update(
+        "foo",
+        3
+    ),
 )
 
 private const val playerSourcePath = "core/player/dist/Player.native.js"
@@ -72,208 +75,119 @@ private fun readResource(path: String): String = classLoader.getResource(path)?.
 
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
-@State(Scope.Benchmark)
-abstract class RuntimePerformance {
+@State(Scope.Thread)
+public abstract class RuntimePerformance {
+
+    @Param(Hermes.name, J2V8.name)
+    protected lateinit var runtimeName: String; private set
 
-    @Param("J2V8", "Graal")
-    lateinit var runtime: String
+    private lateinit var factory: PlayerRuntimeFactory<*>
 
-    lateinit var factory: PlayerRuntimeFactory<*>
+    protected lateinit var runtime: Runtime<*>; private set
 
-    protected fun findRuntimeFactory(runtime: String): PlayerRuntimeFactory<*> =
-        runtimeContainers.single { "$it" == runtime }.factory
+    @Setup public fun setupRuntimeFactory() {
+        factory = findRuntimeFactory(runtimeName)
+    }
 
-    protected fun setupRuntime(runtime: String = this.runtime) {
-        factory = findRuntimeFactory(runtime)
+    protected fun setupRuntime(block: PlayerRuntimeConfig.() -> Unit = {}): Runtime<*> = factory.create(block).also {
+        runtime = it
     }
+
+    private fun findRuntimeFactory(runtimeName: String): PlayerRuntimeFactory<*> =
+        runtimeContainers.single { "$it" == runtimeName }.factory
 }
 
-open class RuntimeCreation : RuntimePerformance() {
+public open class BenchRuntimeCreation : RuntimePerformance() {
 
-    @Setup fun setup() {
-        setupRuntime()
+    @CompilerControl(CompilerControl.Mode.DONT_INLINE)
+    @Benchmark public fun createRuntime(consumer: Blackhole) {
+        // TODO: Somehow this creates a segfault - this only happens very rarely
+        //       but it's really weird. At least it's happening in such a simple
+        //       test. Something might not being tracked when creating the runtime?
+        //       UPDATE: Ohhh, we're attaching the SetTimeoutPlugin. Which registers
+        //       host functions. Great, we'll need to make sure we're guarding those
+        //       appropriately.
+        //       UPDATE 2: Well, the segfault happens on objects, which won't be
+        //       UPDATE 3: Only global & empty are created :( so, tracking ref isn't foolproof?
+        //       UPDATE 4: Reproduced it with value, so now i'm thinking tracking isn't foolproof
+        //       UPDATE 5: Weak ref is killing us :(
+        consumer.consume(setupRuntime())
+        runtime.release()
     }
 
     @CompilerControl(CompilerControl.Mode.DONT_INLINE)
-    @Benchmark fun createRuntime(consumer: Blackhole) {
-        consumer.consume(factory.create())
+    @Benchmark fun createHeadlessPlayer(consumer: Blackhole) {
+        // measures runtime headless player creation + runtime creation
+        // this doesn't technically work the same way it does when the
+        // player usually is created, because it just pulls the first
+        // runtime factory it can find. we use our setupRuntime method
+        // to ensure we release the runtime.
+        consumer.consume(HeadlessPlayer(explicitRuntime = setupRuntime()))
+        // we always release the runtime to make sure jmh isn't waiting
+        // for the runtime thread to be released
+        runtime.release()
     }
-
 }
 
-open class RuntimePlayerCreation : RuntimePerformance() {
-
-    lateinit var jsRuntime: Runtime<*>
+public open class BenchPlayerCreation : RuntimePerformance() {
 
-    @Setup fun setup() {
-        setupRuntime(runtime)
-        jsRuntime = factory.create()
+    // if we don't release the runtime on each bench, we wouldn't need to reload the runtime
+    // on each invocation it might be useful to see how "warm" the runtime can actually get
+    @Setup(Invocation) public fun setup() {
+        // load up runtime and player source in setup to isolate benchmarks
+        setupRuntime()
         playerSource
     }
 
     @CompilerControl(CompilerControl.Mode.DONT_INLINE)
-    @Benchmark fun createHeadlessPlayer(consumer: Blackhole) {
-        consumer.consume(HeadlessPlayer(explicitRuntime = jsRuntime))
+    @Benchmark public fun createJSPlayer(consumer: Blackhole) {
+        consumer.consume(runtime.execute(playerSource))
+        consumer.consume(runtime.execute("""(new Player.Player())"""))
+        runtime.release()
     }
 
     @CompilerControl(CompilerControl.Mode.DONT_INLINE)
-    @Benchmark fun createPlayer(consumer: Blackhole) {
-        jsRuntime.execute(playerSource)
-        consumer.consume(jsRuntime.execute("""(new Player.Player())"""))
+    @Benchmark public fun createHeadlessPlayer(consumer: Blackhole) {
+        // uses runtime created outside benchmark
+        consumer.consume(HeadlessPlayer(explicitRuntime = runtime))
+        runtime.release()
     }
-
 }
 
-open class ContentPerformance : RuntimePerformance() {
-
-    @Param("w2", "pi", "cli")
-    lateinit var content: String
-
-    lateinit var flow: String
-
-    lateinit var update: Update
-
-    lateinit var jsRuntime: Runtime<*>
-
-    protected fun readContent(content: String): String =
-        readResource("$content.json")
+public open class BenchPlayerFlow : RuntimePerformance() {
 
-    protected fun setupContent(content: String = this.content) {
-        setupRuntime()
-        jsRuntime = factory.create()
-        flow = readContent(content)
-        update = mocks[ContentID(content)]!!
-    }
-
-}
+    @Param("mocks/info/info-modal-flow")
+    public lateinit var content: String
 
-open class HeadlessPlayerFlowPerformance : ContentPerformance() {
+    public lateinit var flow: String
 
-    lateinit var runtimeFactory: PlayerRuntimeFactory<*>
+    public lateinit var update: Update
 
-    @Setup fun setup() {
-        setupContent()
+    @Setup(Invocation)
+    public fun setup() {
+        flow = readResource("$content.json")
+        update = mocks[ContentID(content)] ?: throw RuntimeException("update not defined for $content")
     }
 
     @CompilerControl(CompilerControl.Mode.DONT_INLINE)
-    @Benchmark fun startFlow(consumer: Blackhole) {
-        val player = HeadlessPlayer(explicitRuntime = jsRuntime)
-        consumer.consume(runBlocking {
-            suspendCoroutine<Asset?> {
-                player.hooks.viewController.tap("perf") { vc ->
-                    vc?.hooks?.view?.tap("new view") { v ->
-                        v?.hooks?.onUpdate?.tap("test") { a ->
+    @Benchmark
+    fun firstViewUpdate(consumer: Blackhole) {
+        val player = HeadlessPlayer(explicitRuntime = setupRuntime())
+        val pending = player.scope.async {
+            suspendCoroutine {
+                player.hooks.viewController.tap { vc ->
+                    vc?.hooks?.view?.tap { v ->
+                        v?.hooks?.onUpdate?.tap { a ->
                             it.resume(a)
                         }
                     }
                 }
-
-                player.start(flow)
-            }
-        })
-    }
-
-    @CompilerControl(CompilerControl.Mode.DONT_INLINE)
-    @Benchmark fun startAndUpdateFlow(consumer: Blackhole) {
-        val player = HeadlessPlayer(explicitRuntime = jsRuntime)
-        consumer.consume(runBlocking {
-            suspendCoroutine<Asset?> {
-                var dataController: DataController? = null
-                player.hooks.dataController.tap("perf") { dc ->
-                    dataController = dc
-                }
-
-                var updateCount = 0
-                player.hooks.viewController.tap("perf") { vc ->
-                    vc?.hooks?.view?.tap("new view") { v ->
-                        v?.hooks?.onUpdate?.tap("test") { a ->
-                            if (updateCount++ == 0)
-                                dataController!!.set(update.binding to update.value)
-
-                            else it.resume(a)
-                        }
-                    }
-                }
-
                 player.start(flow)
             }
-        })
-    }
-
-}
-
-open class RuntimePlayerFlowPerformance : ContentPerformance() {
-
-    lateinit var runtimeFactory: PlayerRuntimeFactory<*>
-
-    lateinit var flowNode: Node
-
-    @Setup fun setup() {
-        setupContent()
-        flowNode = jsRuntime.execute("""($flow)""") as Node
-    }
-
-    @CompilerControl(CompilerControl.Mode.DONT_INLINE)
-    @Benchmark fun startFlow(consumer: Blackhole) {
-        jsRuntime.execute(playerSource)
-
-        val runner = jsRuntime.execute("""
-((content, playerFactory = () => new Player.Player()) => {
-    return new Promise((resolve, reject) => {
-        try {
-            const player = playerFactory();
-            player.hooks.viewController.tap("perf", (vc) => {
-                vc.hooks.view.tap("new view", (v) => {
-                    v.hooks.onUpdate.tap("test", (a) => {
-                        resolve(a);
-                    });
-                });
-            });
-            player.start(content).catch(reject);
-        } catch (e) { reject(e); }
-    });
-})
-""") as Invokable<Node>
-        val promise = Promise(runner(flowNode))
-
-        consumer.consume(runBlocking {
-            promise.toCompletable<Asset>(Asset.serializer()).await()
-        })
+        }
+        runBlocking {
+            pending.await()
+        }
+        runtime.release()
     }
-
-    @CompilerControl(CompilerControl.Mode.DONT_INLINE)
-    @Benchmark fun startAndUpdateFlow(consumer: Blackhole) {
-        jsRuntime.execute(playerSource)
-
-        val runner = jsRuntime.execute("""
-((content, update, playerFactory = () => new Player.Player()) => {
-    return new Promise((resolve, reject) => {
-        try {
-            const player = playerFactory();
-            let dataController;
-            player.hooks.dataController.tap("perf", (dc) => {
-                dataController = dc;
-            });
-            let updateCount = 0;
-            player.hooks.viewController.tap("perf", (vc) => {
-                vc.hooks.view.tap("new view", (v) => {
-                    v.hooks.onUpdate.tap("test", (a) => {
-                        if (updateCount++ == 0)
-                            dataController.set({ [update.binding]: update.value });
-                        else resolve(a);
-                    });
-                });
-            });
-            player.start(content).catch(reject);
-        } catch (e) { reject(e); }
-    });
-})
-""") as Invokable<Node>
-        val promise = Promise(runner(flowNode, update))
-
-        consumer.consume(runBlocking {
-            promise.toCompletable<Asset>(Asset.serializer()).await()
-        })
-    }
-
 }