Add FrameworkBenchmarks performance test

adoptium · Jun 20, 2022 · 3e2aeea · 3e2aeea
1 parent c671a7f
commit 3e2aeea
Show file tree

Hide file tree

Showing 6 changed files with 199 additions and 22 deletions.
diff --git a/perf/README.md b/perf/README.md
@@ -26,6 +26,7 @@ subdirectory and given a meaningful name.  Once the reorganization of this direc
 //├── dacapo
 //├── renaissance
 //├── liberty
+//├── FrameworkBenchmarks
 
 ```
 Each subdirectory requires a build.xml file describing where to pull the benchmark suite from, and how to build and run it.  Each subdirectory also requires a playlist.xml file which describes 1 or more benchmarks and what commands to run in order to execute a particular benchmark run.
@@ -49,4 +50,7 @@ Liberty benchmarks from https://github.com/OpenLiberty - including liberty-dt7-s
 #### renaissance
 Renaissance benchmarks from https://github.com/renaissance-benchmarks/renaissance - including renaissance-akka-uct, renaissance-als, renaissance-chi-square, renaissance-db-shootout, renaissance-dec-tree, renaissance-finagle-chirper, renaissance-finagle-http, renaissance-fj-kmeans, renaissance-future-genetic, renaissance-gauss-mix, renaissance-log-regression, renaissance-mnemonics, renaissance-movie-lens, renaissance-naive-bayes, renaissance-par-mnemonics, renaissance-philosophers and renaissance-scala-kmeans
 
+#### FrameworkBenchmarks
+FrameworkBenchmarks from https://github.com/TechEmpower/FrameworkBenchmarks - including a wide field of web application frameworks for JAVA.
+
 Additional benchmarks are being reviewed for addition and if you wish to include more, please comment in the open performance benchmarks [issue 1112](https://github.com/adoptium/aqa-tests/issues/1112).
diff --git a/perf/framework-benchmarks/README.md b/perf/framework-benchmarks/README.md
@@ -0,0 +1,26 @@
+# FrameworkBenchmarks
+
+[FrameworkBenchmarks](https://github.com/TechEmpower/FrameworkBenchmarks/tree/master/frameworks/Java)  provides representative performance measures across a wide field of web application frameworks. With much help from the community, coverage is quite broad and we are happy to broaden it further with contributions.
+
+Here we focus on the wide range of Java frameworks for web applications scenarios.
+
+FrameworkBenchmarks covers common workloads as plaintext processing, json content parsing, popular database manipulations 
+for different frameworks. 
+
+Notice the different from external tests:
+```aidl
+external tests runs framework inner unit tests to ensure frameworks behaving correctly.
+perf framework tests stresses common worloads to ensure frameworks are performing well under test VMs.
+```
+
+We are enabling the following list of frameworks to avoid regression:
+
+| Framework | Workloads                              |
+|-----------|----------------------------------------|
+| netty     | plaintext,json                         |
+| quarkus   | plaintext,json,query,fortune,update,db |
+| spark     | plaintext,json,query,fortune,update,db     |
+| spring    | plaintext,json,query,fortune,update,db |
+| spring-jpa  | plaintext,json,query,fortune,update,db |
+| servlet   | plaintext,json                         |
+| servlet3  | plaintext,json                         |
diff --git a/perf/framework-benchmarks/build.xml b/perf/framework-benchmarks/build.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0"?>
+<project name="FrameworkBenchmarks-Test" default="build" basedir=".">
+    <taskdef resource="net/sf/antcontrib/antlib.xml" />
+    <description>
+        Build FrameworkBenchmarks-Test Docker image
+    </description>
+
+    <!-- set properties for this build -->
+    <property name="TEST" value="framework-benchmarks" />
+    <property name="DEST" value="${BUILD_ROOT}/perf/${TEST}" />
+    <property name="src" location="." />
+
+    <condition property="git-prefix" value="git" else="https">
+        <isset property="isZOS"/>
+    </condition>
+
+    <target name="init">
+        <mkdir dir="${DEST}"/>
+    </target>
+
+    <target name="getFrameworkBenchmarks" depends="init" description="Clone the distribution">
+        <echo message="Cloning FrameworkBenchmarks"/>
+        <var name="git_command" value="clone --depth 1 -b master ${git-prefix}://github.com/TechEmpower/FrameworkBenchmarks.git ${DEST}"/>
+        <echo message="git ${git_command}" />
+        <exec executable="git" failonerror="false" dir=".">
+            <arg line="${git_command}" />
+        </exec>
+    </target>
+
+    <target name="dist" depends="getFrameworkBenchmarks" description="generate the distribution">
+        <copy todir="${DEST}">
+            <fileset dir="${src}" includes="*.xml, *.mk"/>
+        </copy>
+    </target>
+
+    <target name="build">
+        <antcall target="dist" inheritall="true" />
+    </target>
+</project>
diff --git a/perf/framework-benchmarks/playlist.xml b/perf/framework-benchmarks/playlist.xml
@@ -0,0 +1,107 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!--
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+<playlist xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../TKG/playlist.xsd">
+	<test>
+		<testCaseName>FrameworkBenchmarks-netty</testCaseName>
+		<command>
+			${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh netty netty.dockerfile ${TEST_RESROOT}; \
+			$(TEST_STATUS)
+		</command>
+		<levels>
+			<level>special</level>
+		</levels>
+		<groups>
+			<group>perf</group>
+		</groups>
+	</test>
+	<test>
+		<testCaseName>FrameworkBenchmarks-quarkus</testCaseName>
+		<command>
+			${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh quarkus quarkus.dockerfile ${TEST_RESROOT}; \
+			$(TEST_STATUS)
+		</command>
+		<levels>
+			<level>special</level>
+		</levels>
+		<groups>
+			<group>perf</group>
+		</groups>
+	</test>
+	<test>
+		<testCaseName>FrameworkBenchmarks-spring</testCaseName>
+		<command>
+			${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh spring spring.dockerfile ${TEST_RESROOT}; \
+			$(TEST_STATUS)
+		</command>
+		<levels>
+			<level>special</level>
+		</levels>
+		<groups>
+			<group>perf</group>
+		</groups>
+	</test>
+	<test>
+		<testCaseName>FrameworkBenchmarks-spring-jpa</testCaseName>
+		<command>
+			${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh spring spring-jpa.dockerfile ${TEST_RESROOT}; \
+			$(TEST_STATUS)
+		</command>
+		<levels>
+			<level>special</level>
+		</levels>
+		<groups>
+			<group>perf</group>
+		</groups>
+	</test>
+	<test>
+		<testCaseName>FrameworkBenchmarks-spark</testCaseName>
+		<command>
+			${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh spark spark.dockerfile ${TEST_RESROOT}; \
+			$(TEST_STATUS)
+		</command>
+		<levels>
+			<level>special</level>
+		</levels>
+		<groups>
+			<group>perf</group>
+		</groups>
+	</test>
+	<test>
+		<testCaseName>FrameworkBenchmarks-servlet3</testCaseName>
+		<command>
+			${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh servlet3 servlet3.dockerfile ${TEST_RESROOT}; \
+			$(TEST_STATUS)
+		</command>
+		<levels>
+			<level>special</level>
+		</levels>
+		<groups>
+			<group>perf</group>
+		</groups>
+	</test>
+	<test>
+		<testCaseName>FrameworkBenchmarks-servlet</testCaseName>
+		<command>
+			${TEST_RESROOT}/../../../aqa-tests/perf/tfb-runner.sh servlet servlet.dockerfile ${TEST_RESROOT}; \
+			$(TEST_STATUS)
+		</command>
+		<levels>
+			<level>special</level>
+		</levels>
+		<groups>
+			<group>perf</group>
+		</groups>
+	</test>
+</playlist>
diff --git a/perf/tfb-runner.sh b/perf/tfb-runner.sh
@@ -0,0 +1,22 @@
+test=$1
+dockerfile=$2
+TEST_RESROOT=$3
+
+cd ${TEST_RESROOT}
+testcase=` echo ${dockerfile} | cut -d'.' -f 1`
+dockerfile=${TEST_RESROOT}/frameworks/Java/${test}/${dockerfile}
+
+if [ ! -f ${dockerfile} ]; then
+  echo "Warning: ${dockerfile} not found"
+  exit 1
+fi
+
+cp -r ${TEST_JDK_HOME} ${TEST_RESROOT}/frameworks/Java/${test}/jdk
+awk '/CMD.*/ && !x {print "COPY jdk /opt/java/openjdk"; x=1} 1' ${dockerfile} > tmpt && mv tmpt ${dockerfile}
+awk '/CMD.*/ && !x {print "ENV JAVA_HOME=/opt/java/openjdk"; x=1} 1' ${dockerfile} > tmpt && mv tmpt ${dockerfile}
+awk '/CMD.*/ && !x {print "ENV PATH=/opt/java/openjdk/bin:$PATH"; x=1} 1' ${dockerfile} > tmpt && mv tmpt ${dockerfile}
+cat ${dockerfile}
+./tfb --test $testcase
+
+# get removable results
+docker run --entrypoint /bin/bash  --rm --network tfb -v /var/run/docker.sock:/var/run/docker.sock -v ${TEST_RESROOT}:/FrameworkBenchmarks techempower/tfb -c "chmod -R 777 /FrameworkBenchmarks/results"
diff --git a/testenv/testenv.properties b/testenv/testenv.properties
@@ -1,23 +1,2 @@
-TKG_REPO=https://github.com/adoptium/TKG.git
-TKG_BRANCH=master
 OPENJ9_REPO=https://github.com/eclipse-openj9/openj9.git
-OPENJ9_BRANCH=master
-STF_REPO=https://github.com/adoptium/STF.git
-STF_BRANCH=master
-OPENJ9_SYSTEMTEST_REPO=https://github.com/eclipse-openj9/openj9-systemtest.git
-OPENJ9_SYSTEMTEST_BRANCH=master
-ADOPTOPENJDK_SYSTEMTEST_REPO=https://github.com/adoptium/aqa-systemtest.git
-ADOPTOPENJDK_SYSTEMTEST_BRANCH=master
-JDK8_REPO=https://github.com/adoptium/jdk8u.git
-JDK8_BRANCH=dev
-JDK11_REPO=https://github.com/adoptium/jdk11u.git
-JDK11_BRANCH=dev
-JDK17_REPO=https://github.com/adoptium/jdk17u.git
-JDK17_BRANCH=dev
-JDK8_OPENJ9_REPO=https://github.com/ibmruntimes/openj9-openjdk-jdk8.git
-JDK8_OPENJ9_BRANCH=openj9
-JDK11_OPENJ9_REPO=https://github.com/ibmruntimes/openj9-openjdk-jdk11.git
-JDK11_OPENJ9_BRANCH=openj9
-JDK17_OPENJ9_REPO=https://github.com/ibmruntimes/openj9-openjdk-jdk17.git
-JDK17_OPENJ9_BRANCH=openj9
-AQA_REQUIRED_TARGETS=sanity.functional,extended.functional,special.functional,sanity.openjdk,extended.openjdk,sanity.system,extended.system,sanity.perf,extended.perf
+OPENJ9_SHA=b8b68fd9224ddb378cdfa2e1fb6cdd86aac62eb1