diff --git a/perf/README.md b/perf/README.md index b36c4584b2..f860336c58 100644 --- a/perf/README.md +++ b/perf/README.md @@ -26,6 +26,7 @@ subdirectory and given a meaningful name. Once the reorganization of this direc //├── dacapo //├── renaissance //├── liberty +//├── FrameworkBenchmarks ``` Each subdirectory requires a build.xml file describing where to pull the benchmark suite from, and how to build and run it. Each subdirectory also requires a playlist.xml file which describes 1 or more benchmarks and what commands to run in order to execute a particular benchmark run. @@ -49,4 +50,7 @@ Liberty benchmarks from https://github.com/OpenLiberty - including liberty-dt7-s #### renaissance Renaissance benchmarks from https://github.com/renaissance-benchmarks/renaissance - including renaissance-akka-uct, renaissance-als, renaissance-chi-square, renaissance-db-shootout, renaissance-dec-tree, renaissance-finagle-chirper, renaissance-finagle-http, renaissance-fj-kmeans, renaissance-future-genetic, renaissance-gauss-mix, renaissance-log-regression, renaissance-mnemonics, renaissance-movie-lens, renaissance-naive-bayes, renaissance-par-mnemonics, renaissance-philosophers and renaissance-scala-kmeans +#### FrameworkBenchmarks +FrameworkBenchmarks from https://github.com/TechEmpower/FrameworkBenchmarks - including a wide field of web application frameworks for JAVA. + Additional benchmarks are being reviewed for addition and if you wish to include more, please comment in the open performance benchmarks [issue 1112](https://github.com/adoptium/aqa-tests/issues/1112). diff --git a/perf/framework-benchmarks/README.md b/perf/framework-benchmarks/README.md new file mode 100644 index 0000000000..af05b03106 --- /dev/null +++ b/perf/framework-benchmarks/README.md @@ -0,0 +1,26 @@ +# FrameworkBenchmarks + +[FrameworkBenchmarks](https://github.com/TechEmpower/FrameworkBenchmarks/tree/master/frameworks/Java) provides representative performance measures across a wide field of web application frameworks. With much help from the community, coverage is quite broad and we are happy to broaden it further with contributions. + +Here we focus on the wide range of Java frameworks for web applications scenarios. + +FrameworkBenchmarks covers common workloads as plaintext processing, json content parsing, popular database manipulations +for different frameworks. + +Notice the different from external tests: +```aidl +external tests runs framework inner unit tests to ensure frameworks behaving correctly. +perf framework tests stresses common worloads to ensure frameworks are performing well under test VMs. +``` + +We are enabling the following list of frameworks to avoid regression: + +| Framework | Workloads | +|-----------|----------------------------------------| +| netty | plaintext,json | +| quarkus | plaintext,json,query,fortune,update,db | +| spark | plaintext,json,query,fortune,update,db | +| spring | plaintext,json,query,fortune,update,db | +| spring-jpa | plaintext,json,query,fortune,update,db | +| servlet | plaintext,json | +| servlet3 | plaintext,json | diff --git a/perf/framework-benchmarks/build.xml b/perf/framework-benchmarks/build.xml new file mode 100644 index 0000000000..d37e4da350 --- /dev/null +++ b/perf/framework-benchmarks/build.xml @@ -0,0 +1,39 @@ + + + + + Build FrameworkBenchmarks-Test Docker image + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/perf/framework-benchmarks/playlist.xml b/perf/framework-benchmarks/playlist.xml new file mode 100644 index 0000000000..8fb7762a6c --- /dev/null +++ b/perf/framework-benchmarks/playlist.xml @@ -0,0 +1,135 @@ + + + + + FrameworkBenchmarks-netty + + ${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh netty netty.dockerfile ${TEST_RESROOT}; \ + $(TEST_STATUS) + + + special + + + perf + + + 11+ + + os.linux,arch.x86,bits.64 + + + FrameworkBenchmarks-quarkus + + ${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh quarkus quarkus.dockerfile ${TEST_RESROOT}; \ + $(TEST_STATUS) + + + special + + + perf + + + 11+ + + os.linux,arch.x86,bits.64 + + + FrameworkBenchmarks-spring + + ${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh spring spring.dockerfile ${TEST_RESROOT}; \ + $(TEST_STATUS) + + + special + + + perf + + + 11+ + + os.linux,arch.x86,bits.64 + + + FrameworkBenchmarks-spring-jpa + + ${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh spring spring-jpa.dockerfile ${TEST_RESROOT}; \ + $(TEST_STATUS) + + + special + + + perf + + + 11+ + + os.linux,arch.x86,bits.64 + + + FrameworkBenchmarks-spark + + ${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh spark spark.dockerfile ${TEST_RESROOT}; \ + $(TEST_STATUS) + + + special + + + perf + + + 11+ + + os.linux,arch.x86,bits.64 + + + FrameworkBenchmarks-servlet3 + + ${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh servlet3 servlet3.dockerfile ${TEST_RESROOT}; \ + $(TEST_STATUS) + + + special + + + perf + + + 11+ + + os.linux,arch.x86,bits.64 + + + FrameworkBenchmarks-servlet + + ${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh servlet servlet.dockerfile ${TEST_RESROOT}; \ + $(TEST_STATUS) + + + special + + + perf + + + 11+ + + os.linux,arch.x86,bits.64 + + diff --git a/perf/tfb-runner.sh b/perf/tfb-runner.sh new file mode 100755 index 0000000000..34cf8dee25 --- /dev/null +++ b/perf/tfb-runner.sh @@ -0,0 +1,22 @@ +test=$1 +dockerfile=$2 +TEST_RESROOT=$3 + +cd ${TEST_RESROOT} +testcase=` echo ${dockerfile} | cut -d'.' -f 1` +dockerfile=${TEST_RESROOT}/frameworks/Java/${test}/${dockerfile} + +if [ ! -f ${dockerfile} ]; then + echo "Warning: ${dockerfile} not found" + exit 1 +fi + +cp -r ${TEST_JDK_HOME} ${TEST_RESROOT}/frameworks/Java/${test}/jdk +awk '/CMD.*/ && !x {print "COPY jdk /opt/java/openjdk"; x=1} 1' ${dockerfile} > tmpt && mv tmpt ${dockerfile} +awk '/CMD.*/ && !x {print "ENV JAVA_HOME=/opt/java/openjdk"; x=1} 1' ${dockerfile} > tmpt && mv tmpt ${dockerfile} +awk '/CMD.*/ && !x {print "ENV PATH=/opt/java/openjdk/bin:$PATH"; x=1} 1' ${dockerfile} > tmpt && mv tmpt ${dockerfile} +cat ${dockerfile} +./tfb --test $testcase + +# get removable results +docker run --entrypoint /bin/bash --rm --network tfb -v /var/run/docker.sock:/var/run/docker.sock -v ${TEST_RESROOT}:/FrameworkBenchmarks techempower/tfb -c "chmod -R 777 /FrameworkBenchmarks/results" \ No newline at end of file