Add benchmark filter run script (#3271)

* Added benchmark run script

Signed-off-by: Calum Murray <[email protected]>

* Updated docs to explain how to run filter benchmarks

Signed-off-by: Calum Murray <[email protected]>

* Add time in unix to each benchmark run output file

Signed-off-by: Calum Murray <[email protected]>

* Actually use the logging config file

Signed-off-by: Calum Murray <[email protected]>

* Added check to see if a valid classname is provided

Signed-off-by: Calum Murray <[email protected]>

* Added list of available java benchmark classes to README

Signed-off-by: Calum Murray <[email protected]>

---------

Signed-off-by: Calum Murray <[email protected]>

.gitignore

@@ -45,3 +45,6 @@ data-plane/**/build/
 /eventing-kafka-source-bundle.yaml
 
 /eventing-kafka-tls-networking.yaml
+
+## Ignore benchmark outputs
+data-plane/**/*.out.txt

DEVELOPMENT.md

@@ -164,6 +164,27 @@ export EVENT=alloc
 
 For more information on the profiler test, see [the profiler test doc](./data-plane/profiler/README.md).
 
+### Run Filter Benchmarks
+
+If you are building a fitler benchmark or want to benchmark the performance delta caused when changing the filters, you can run:
+
+```shell
+./hack/run.sh benchmark-filter <filter_class_name>
+```
+
+This will run the benchmarks for the class with `<filter_class_name>`. A full list of all available classes can be seen [here](https://github.com/knative-extensions/eventing-kafka-broker/blob/main/data-plane/benchmarks/resources/filter-class-list.txt).
+For example, if you want to run all of the Exact Filter Benchmarks, you could run:
+
+```shell
+./hack/run.sh benchmark-filter ExactFilterBenchmark
+```
+
+Alternatively, if you want to run all of the benchmarks you can run:
+
+```shell
+./hack/run.sh benchmark-filters
+```
+
 ## Code generation
 
 Sometimes, before deploying the services it's required to run our code generators, by running the following command:

data-plane/benchmarks/resources/config-logging.xml

@@ -0,0 +1,25 @@
+<!--
+
+    Copyright © 2018 Knative Authors ([email protected])
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<configuration>
+  <appender name="jsonConsoleAppender" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
+  </appender>
+  <root level="INFO">
+    <appender-ref ref="jsonConsoleAppender"/>
+  </root>
+</configuration>

data-plane/benchmarks/resources/filter-class-list.txt

@@ -0,0 +1 @@
+ExactFilterBenchmark

data-plane/benchmarks/run.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# Copyright © 2018 Knative Authors ([email protected])
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+TIME=$(date +%s)
+
+run_java_filter_benchmarks_for_class() {
+  CLASS=$1
+
+  echo "Running benchmarks for ${CLASS}"
+
+  java -Dlogback.configurationFile=${SCRIPT_DIR}/resources/config-logging.xml \
+    -jar "${SCRIPT_DIR}/target/benchmarks.jar" $CLASS 2>&1 | tee "${SCRIPT_DIR}/output/${CLASS}.${TIME}.out.txt"
+
+  echo "Successfully ran benchmarks for ${CLASS}!\n\nThe results can be found at ${SCRIPT_DIR}/output/${CLASS}.${TIME}.out.txt"
+}
+
+SCRIPT_DIR=$(cd -- "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+
+DATA_PLANE_DIR="${SCRIPT_DIR}/../"
+
+# check if the benchmark class exists
+
+if [[ ! -z $1 ]]; then
+  FOUND=0
+  while IFS="" read -r p || [[ -n "$p" ]]; do
+    if [[ "$1" == "$p" ]]; then
+      FOUND=1
+      break
+    fi
+  done <"${SCRIPT_DIR}/resources/filter-class-list.txt"
+  if [[ "$FOUND" != 1 ]]; then
+    echo "Please provide a valid class name for a filter benchmark"
+    exit 1
+  fi
+fi
+
+pushd ${DATA_PLANE_DIR} || return $?
+
+# build only benchmarks and it's dependents - skip aggregating licenses as it will be missing licenses due to only
+# building some projects
+./mvnw clean package -DskipTests -Dlicense.skipAggregateDownloadLicenses -Dlicense.skipAggregateAddThirdParty -P no-release -pl benchmarks -am
+
+popd || return $?
+
+mkdir -p "${SCRIPT_DIR}/output"
+
+if [[ -z $1 ]]; then
+  while IFS="" read -r p || [ -n "$p" ]; do
+    run_java_filter_benchmarks_for_class "$p"
+  done <"${SCRIPT_DIR}/resources/filter-class-list.txt"
+else
+  run_java_filter_benchmarks_for_class $1
+fi

hack/run.sh

@@ -34,6 +34,8 @@ function usage() {
   echo "   generate                                                Run code generators"
   echo "   build-from-source                                       Build artifacts from source"
   echo "   build-for-source-from-source                            Build artifacts from source for source bundle only"
+  echo "   benchmark-filter <bencmark_class_name>                  Run the filter benchmarks for <benchmark_class_name>"
+  echo "   benchmark-filters                                       Run all the filter benchmarks"
   echo ""
 }
 
@@ -87,6 +89,10 @@ elif [[ "${action}" == "profiler" ]]; then
 elif [[ "${action}" == "generate" ]]; then
   "${ROOT_DIR}"/hack/generate-proto.sh
   "${ROOT_DIR}"/hack/update-codegen.sh
+elif [[ "${action}" == "benchmark-filter" ]]; then
+  "${ROOT_DIR}/data-plane/benchmarks/run.sh" "$2"
+elif [[ "${action}" == "benchmark-filters" ]]; then
+  "${ROOT_DIR}/data-plane/benchmarks/run.sh"
 else
   echo "Unrecognized action ${action}"
   usage "$0"