From ee94e9ad4202c89c1582f3af74f841fc0eb2b2e3 Mon Sep 17 00:00:00 2001 From: Calum Murray Date: Fri, 4 Aug 2023 14:23:32 -0400 Subject: [PATCH 1/6] Added benchmark run script Signed-off-by: Calum Murray --- .gitignore | 3 ++ .../benchmarks/resources/config-logging.xml | 25 ++++++++++ .../resources/filter-class-list.txt | 1 + data-plane/benchmarks/run.sh | 49 +++++++++++++++++++ hack/run.sh | 6 +++ 5 files changed, 84 insertions(+) create mode 100644 data-plane/benchmarks/resources/config-logging.xml create mode 100644 data-plane/benchmarks/resources/filter-class-list.txt create mode 100755 data-plane/benchmarks/run.sh diff --git a/.gitignore b/.gitignore index 2d4d7d9de1..f573f413d7 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ data-plane/**/build/ /eventing-kafka-source-bundle.yaml /eventing-kafka-tls-networking.yaml + +## Ignore benchmark outputs +data-plane/**/*.out.txt diff --git a/data-plane/benchmarks/resources/config-logging.xml b/data-plane/benchmarks/resources/config-logging.xml new file mode 100644 index 0000000000..19828c5f4c --- /dev/null +++ b/data-plane/benchmarks/resources/config-logging.xml @@ -0,0 +1,25 @@ + + + + + + + + + diff --git a/data-plane/benchmarks/resources/filter-class-list.txt b/data-plane/benchmarks/resources/filter-class-list.txt new file mode 100644 index 0000000000..3f7b7ddc9a --- /dev/null +++ b/data-plane/benchmarks/resources/filter-class-list.txt @@ -0,0 +1 @@ +ExactFilterBenchmark diff --git a/data-plane/benchmarks/run.sh b/data-plane/benchmarks/run.sh new file mode 100755 index 0000000000..cdf1a18bb1 --- /dev/null +++ b/data-plane/benchmarks/run.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Copyright © 2018 Knative Authors (knative-dev@googlegroups.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +run_java_filter_benchmarks_for_class() { + CLASS=$1 + + echo "Running benchmarks for ${CLASS}" + + java -jar "${SCRIPT_DIR}/target/benchmarks.jar" $CLASS 2>&1 | tee "${SCRIPT_DIR}/output/${CLASS}.out.txt" + + echo "Successfully ran benchmarks for ${CLASS}!\n\nThe results can be found at ${SCRIPT_DIR}/output/${CLASS}.out.txt" +} + +SCRIPT_DIR=$(cd -- "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +DATA_PLANE_DIR="${SCRIPT_DIR}/../" + +pushd ${DATA_PLANE_DIR} || return $? + +# build only benchmarks and it's dependents - skip aggregating licenses as it will be missing licenses due to only +# building some projects +./mvnw clean package -DskipTests -Dlicense.skipAggregateDownloadLicenses -Dlicense.skipAggregateAddThirdParty -P no-release -pl benchmarks -am + +popd || return $? + +mkdir -p "${SCRIPT_DIR}/output" + +if [[ -z $1 ]]; then + while IFS="" read -r p || [ -n "$p" ]; do + run_java_filter_benchmarks_for_class "$p" + done <"${SCRIPT_DIR}/resources/filter-class-list.txt" +else + run_java_filter_benchmarks_for_class $1 +fi diff --git a/hack/run.sh b/hack/run.sh index 0d4b658b3a..0e30d0ca7b 100755 --- a/hack/run.sh +++ b/hack/run.sh @@ -34,6 +34,8 @@ function usage() { echo " generate Run code generators" echo " build-from-source Build artifacts from source" echo " build-for-source-from-source Build artifacts from source for source bundle only" + echo " benchmark-filter Run the filter benchmarks for " + echo " benchmark-filters Run all the filter benchmarks" echo "" } @@ -87,6 +89,10 @@ elif [[ "${action}" == "profiler" ]]; then elif [[ "${action}" == "generate" ]]; then "${ROOT_DIR}"/hack/generate-proto.sh "${ROOT_DIR}"/hack/update-codegen.sh +elif [[ "${action}" == "benchmark-filter" ]]; then + "${ROOT_DIR}/data-plane/benchmarks/run.sh" "$2" +elif [[ "${action}" == "benchmark-filters" ]]; then + "${ROOT_DIR}/data-plane/benchmarks/run.sh" else echo "Unrecognized action ${action}" usage "$0" From 83e8b880b03e1205d3388c14e9cf0edc4fb8a2ff Mon Sep 17 00:00:00 2001 From: Calum Murray Date: Fri, 4 Aug 2023 14:27:07 -0400 Subject: [PATCH 2/6] Updated docs to explain how to run filter benchmarks Signed-off-by: Calum Murray --- DEVELOPMENT.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 249bb6e3a7..e84da93695 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -164,6 +164,26 @@ export EVENT=alloc For more information on the profiler test, see [the profiler test doc](./data-plane/profiler/README.md). +### Run Filter Benchmarks + +If you are building a fitler benchmark or want to benchmark the performance delta caused when changing the filters, you can run: + +```shell +./hack/run.sh benchmark-filter +``` + +This will run the benchmarks for the class with ``. For example, if you want to run all of the Exact Filter Benchmarks, you could run: + +```shell +./hack/run.sh benchmark-filter ExactFilterBenchmark +``` + +Alternatively, if you want to run all of the benchmarks you can run: + +```shell +./hack/run.sh benchmark-filters +``` + ## Code generation Sometimes, before deploying the services it's required to run our code generators, by running the following command: From 58c3713008003e1e88f9e51be47d415b70791bb5 Mon Sep 17 00:00:00 2001 From: Calum Murray Date: Tue, 8 Aug 2023 11:31:22 -0400 Subject: [PATCH 3/6] Add time in unix to each benchmark run output file Signed-off-by: Calum Murray --- data-plane/benchmarks/run.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/data-plane/benchmarks/run.sh b/data-plane/benchmarks/run.sh index cdf1a18bb1..ef3413556b 100755 --- a/data-plane/benchmarks/run.sh +++ b/data-plane/benchmarks/run.sh @@ -16,14 +16,16 @@ set -e +TIME=$(date +%s) + run_java_filter_benchmarks_for_class() { CLASS=$1 echo "Running benchmarks for ${CLASS}" - java -jar "${SCRIPT_DIR}/target/benchmarks.jar" $CLASS 2>&1 | tee "${SCRIPT_DIR}/output/${CLASS}.out.txt" + java -jar "${SCRIPT_DIR}/target/benchmarks.jar" $CLASS 2>&1 | tee "${SCRIPT_DIR}/output/${CLASS}.${TIME}.out.txt" - echo "Successfully ran benchmarks for ${CLASS}!\n\nThe results can be found at ${SCRIPT_DIR}/output/${CLASS}.out.txt" + echo "Successfully ran benchmarks for ${CLASS}!\n\nThe results can be found at ${SCRIPT_DIR}/output/${CLASS}.${TIME}.out.txt" } SCRIPT_DIR=$(cd -- "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) From 5cc64943d4c13f38d24570d47bb2ba8225b96012 Mon Sep 17 00:00:00 2001 From: Calum Murray Date: Tue, 8 Aug 2023 14:34:34 -0400 Subject: [PATCH 4/6] Actually use the logging config file Signed-off-by: Calum Murray --- data-plane/benchmarks/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data-plane/benchmarks/run.sh b/data-plane/benchmarks/run.sh index ef3413556b..32d7e1f23c 100755 --- a/data-plane/benchmarks/run.sh +++ b/data-plane/benchmarks/run.sh @@ -23,7 +23,8 @@ run_java_filter_benchmarks_for_class() { echo "Running benchmarks for ${CLASS}" - java -jar "${SCRIPT_DIR}/target/benchmarks.jar" $CLASS 2>&1 | tee "${SCRIPT_DIR}/output/${CLASS}.${TIME}.out.txt" + java -Dlogback.configurationFile=${SCRIPT_DIR}/resources/config-logging.xml \ + -jar "${SCRIPT_DIR}/target/benchmarks.jar" $CLASS 2>&1 | tee "${SCRIPT_DIR}/output/${CLASS}.${TIME}.out.txt" echo "Successfully ran benchmarks for ${CLASS}!\n\nThe results can be found at ${SCRIPT_DIR}/output/${CLASS}.${TIME}.out.txt" } From 238add94b7e39b13ce68ae5d9b12227114836c9c Mon Sep 17 00:00:00 2001 From: Calum Murray Date: Thu, 10 Aug 2023 15:44:24 -0400 Subject: [PATCH 5/6] Added check to see if a valid classname is provided Signed-off-by: Calum Murray --- data-plane/benchmarks/run.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/data-plane/benchmarks/run.sh b/data-plane/benchmarks/run.sh index 32d7e1f23c..f0d39f3713 100755 --- a/data-plane/benchmarks/run.sh +++ b/data-plane/benchmarks/run.sh @@ -33,6 +33,22 @@ SCRIPT_DIR=$(cd -- "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd) DATA_PLANE_DIR="${SCRIPT_DIR}/../" +# check if the benchmark class exists + +if [[ ! -z $1 ]]; then + FOUND=0 + while IFS="" read -r p || [[ -n "$p" ]]; do + if [[ "$1" == "$p" ]]; then + FOUND=1 + break + fi + done <"${SCRIPT_DIR}/resources/filter-class-list.txt" + if [[ "$FOUND" != 1 ]]; then + echo "Please provide a valid class name for a filter benchmark" + exit 1 + fi +fi + pushd ${DATA_PLANE_DIR} || return $? # build only benchmarks and it's dependents - skip aggregating licenses as it will be missing licenses due to only From b2623325bdad674f5ae0ee6996176c8924686306 Mon Sep 17 00:00:00 2001 From: Calum Murray Date: Tue, 15 Aug 2023 10:16:12 -0400 Subject: [PATCH 6/6] Added list of available java benchmark classes to README Signed-off-by: Calum Murray --- DEVELOPMENT.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e84da93695..22a04780df 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -172,7 +172,8 @@ If you are building a fitler benchmark or want to benchmark the performance delt ./hack/run.sh benchmark-filter ``` -This will run the benchmarks for the class with ``. For example, if you want to run all of the Exact Filter Benchmarks, you could run: +This will run the benchmarks for the class with ``. A full list of all available classes can be seen [here](https://github.com/knative-extensions/eventing-kafka-broker/blob/main/data-plane/benchmarks/resources/filter-class-list.txt). +For example, if you want to run all of the Exact Filter Benchmarks, you could run: ```shell ./hack/run.sh benchmark-filter ExactFilterBenchmark