From 40b9e677f1f8594b341e7352776e0971049351bf Mon Sep 17 00:00:00 2001 From: Michal Karm Babacek Date: Mon, 9 Sep 2024 17:16:29 +0200 Subject: [PATCH 1/4] Tests config for serialization of arrays --- apps/for-serialization/Dockerfile | 5 + apps/for-serialization/pom.xml | 39 +++++ .../src/main/java/for_serialization/Main.java | 165 ++++++++++++++++++ pom.xml | 1 + .../tests/integration/AppReproducersTest.java | 65 ++++++- .../graalvm/tests/integration/utils/Apps.java | 15 +- .../integration/utils/BuildAndRunCmds.java | 35 ++++ .../integration/utils/ContainerNames.java | 1 + .../integration/utils/WhitelistLogLines.java | 8 + 9 files changed, 328 insertions(+), 6 deletions(-) create mode 100644 apps/for-serialization/Dockerfile create mode 100644 apps/for-serialization/pom.xml create mode 100644 apps/for-serialization/src/main/java/for_serialization/Main.java diff --git a/apps/for-serialization/Dockerfile b/apps/for-serialization/Dockerfile new file mode 100644 index 0000000..dc84aaa --- /dev/null +++ b/apps/for-serialization/Dockerfile @@ -0,0 +1,5 @@ +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10 +WORKDIR /work/ +RUN chown 100 /work \ + && chmod "g+rwX" /work \ + && chown 1000:root /work diff --git a/apps/for-serialization/pom.xml b/apps/for-serialization/pom.xml new file mode 100644 index 0000000..f6ba1dd --- /dev/null +++ b/apps/for-serialization/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + for-serialization + for-serialization + 1 + + for-serialization + + + org.graalvm.tests.integration + parent + 1.0.0-SNAPSHOT + ../../pom.xml + + + + 3.2.0 + + + + for-serialization + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + for_serialization.Main + + + + + + + diff --git a/apps/for-serialization/src/main/java/for_serialization/Main.java b/apps/for-serialization/src/main/java/for_serialization/Main.java new file mode 100644 index 0000000..2ff691f --- /dev/null +++ b/apps/for-serialization/src/main/java/for_serialization/Main.java @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2024, Red Hat Inc. All rights reserved. + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package for_serialization; + +import sun.reflect.ReflectionFactory; + +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.Serial; +import java.io.Serializable; +import java.lang.reflect.Array; +import java.lang.reflect.Constructor; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * @author Michal Karm Babacek + * Inspired by https://github.com/oracle/graal/issues/8509 + * Tests https://github.com/oracle/graal/issues/9581 + */ +public class Main { + + static class Muhehehe implements Serializable { + @Serial + private static final long serialVersionUID = 5197858094838069415L; + } + + class MuheheheNested implements Serializable { + @Serial + private static final long serialVersionUID = 6197858094838069415L; + + class MuheheheNestedNested implements Serializable { + @Serial + private static final long serialVersionUID = 7197858094838069415L; + } + } + + public static void main(String[] args) throws Exception { + final Class[] arrayTypes = new Class[] { + boolean[].class, + boolean[][].class, + byte[].class, + byte[][].class, + char[].class, + char[][].class, + double[].class, + double[][].class, + float[].class, + float[][].class, + int[].class, + int[][].class, + long[].class, + long[][].class, + short[].class, + short[][].class, + Boolean[].class, + Boolean[][].class, + Byte[].class, + Byte[][].class, + Character[].class, + Character[][].class, + Double[].class, + Double[][].class, + Float[].class, + Float[][].class, + Integer[].class, + Integer[][].class, + Long[].class, + Long[][].class, + Short[].class, + Short[][].class, + // https://github.com/oracle/graal/issues/9581 + String[].class, + String[][].class, + Void[].class, + Void[][].class, + Class[].class, + Class[][].class, + Object[].class, + Object[][].class, + Number[].class, + Number[][].class, + Comparable[].class, + Comparable[][].class, + Serializable[].class, + Serializable[][].class, + Cloneable[].class, + Cloneable[][].class, + Readable[].class, + Readable[][].class, + AutoCloseable[].class, + AutoCloseable[][].class, + Closeable[].class, + Closeable[][].class, + Appendable[].class, + Appendable[][].class, + PrintStream[].class, + PrintStream[][].class, + PrintWriter[].class, + PrintWriter[][].class, + Record[].class, + Record[][].class, + Enum[].class, + Enum[][].class, + Enum[][][].class, + Enum[][][][].class, + Enum[][][][][].class, + Muhehehe[].class, + Muhehehe[][].class, + Muhehehe[][][].class, + MuheheheNested[].class, + MuheheheNested[][].class, + MuheheheNested[][][].class, + MuheheheNested.MuheheheNestedNested[].class, + MuheheheNested.MuheheheNestedNested[][].class, + MuheheheNested.MuheheheNestedNested[][][].class + }; + + final File f = Files.createTempFile(Path.of("."), "tmp", ".ser").toFile(); + f.deleteOnExit(); + for (int i = 0; i < arrayTypes.length; i++) { + final Constructor cons = ReflectionFactory.getReflectionFactory() + .newConstructorForSerialization(arrayTypes[i], + java.lang.Object.class.getDeclaredConstructor((Class[]) null)); + final Object o = Array.newInstance(arrayTypes[i].getComponentType(), 2); + try (final ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(f)); + final ObjectInputStream ois = new ObjectInputStream(new FileInputStream(f))) { + oos.writeObject(o); + oos.flush(); + final Object o2 = ois.readObject(); + System.out.printf("%d %d %s %s %b\n", + i, cons.getParameterCount(), o.getClass().descriptorString(), o2.getClass().descriptorString(), + o2.getClass().equals(o.getClass())); + } catch (IOException e) { + e.printStackTrace(); + } + } + System.out.println("\nDone."); + } +} + diff --git a/pom.xml b/pom.xml index bea0369..2e0fa88 100755 --- a/pom.xml +++ b/pom.xml @@ -100,6 +100,7 @@ apps/timezones apps/versions apps/monitor-field-offset + apps/for-serialization testsuite diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java b/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java index 7ccc5ae..81331c5 100644 --- a/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java +++ b/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java @@ -42,7 +42,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -497,7 +496,7 @@ public void imageioAWTContainerTest(TestInfo testInfo) throws IOException, Inter @Test @Tag("imageio") - @DisabledOnOs({OS.WINDOWS, OS.MAC}) // AWT support is not there yet + @DisabledOnOs({ OS.WINDOWS, OS.MAC }) // AWT support is not there yet @IfMandrelVersion(min = "21.1") public void imageioAWTTest(TestInfo testInfo) throws IOException, InterruptedException { imageioAWT(testInfo, Apps.IMAGEIO); @@ -818,6 +817,68 @@ public void monitorFieldOffset(TestInfo testInfo, Apps app) throws IOException, } } + @Test + @Tag("builder-image") + @IfMandrelVersion(min = "23.1.5", inContainer = true) + public void forSerializationContainerTest(TestInfo testInfo) throws IOException, InterruptedException { + forSerialization(testInfo, Apps.FOR_SERIALIZATION_BUILDER_IMAGE); + } + + @Test + @IfMandrelVersion(minJDK = "23.1.5") + public void forSerializationTest(TestInfo testInfo) throws IOException, InterruptedException { + forSerialization(testInfo, Apps.FOR_SERIALIZATION); + } + + public void forSerialization(TestInfo testInfo, Apps app) throws IOException, InterruptedException { + LOGGER.info("Testing app: " + app); + File processLog = null; + final StringBuilder report = new StringBuilder(); + final File appDir = Path.of(BASE_DIR, app.dir).toFile(); + final File metaINF = Path.of(BASE_DIR, app.dir, "src", "main", "resources", "META-INF", "native-image").toFile(); + final String cn = testInfo.getTestClass().get().getCanonicalName(); + final String mn = testInfo.getTestMethod().get().getName(); + final boolean inContainer = app == Apps.FOR_SERIALIZATION_BUILDER_IMAGE; + try { + // Cleanup + cleanTarget(app); + if (metaINF.exists()) { + FileUtils.cleanDirectory(metaINF); + } + if (inContainer) { + removeContainers(app.runtimeContainer.name); + } + Files.createDirectories(Paths.get(appDir.getAbsolutePath() + File.separator + "logs")); + processLog = Path.of(appDir.getAbsolutePath(), "logs", "build-and-run.log").toFile(); + builderRoutine(inContainer ? 4 : 3, app, report, cn, mn, appDir, processLog); + LOGGER.info("Running..."); + + final List cmdHotSpot = getRunCommand(app.buildAndRunCmds.cmds[app.buildAndRunCmds.cmds.length - 2]); + final List cmdNative = getRunCommand(app.buildAndRunCmds.cmds[app.buildAndRunCmds.cmds.length - 1]); + final String hotSpotOutput = runCommand(cmdHotSpot, appDir); + final String nativeOutput = runCommand(cmdNative, appDir); + Logs.appendln(report, appDir.getAbsolutePath()); + Logs.appendlnSection(report, String.join(" ", cmdHotSpot)); + Logs.appendlnSection(report, hotSpotOutput); + Logs.appendlnSection(report, String.join(" ", cmdNative)); + Logs.appendlnSection(report, nativeOutput); + + assertEquals(hotSpotOutput, nativeOutput, "The output of the HotSpot and native-image runs must be the same."); + + Logs.checkLog(cn, mn, app, processLog); + + if (inContainer) { + removeContainers(app.runtimeContainer.name); + } + + } finally { + cleanup(null, cn, mn, report, app, processLog); + if (metaINF.exists()) { + FileUtils.cleanDirectory(metaINF); + } + } + } + @Test @Tag("calendars") @IfMandrelVersion(min = "22.3.5") // The fix for this test is in 22.3.5 and better diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/utils/Apps.java b/testsuite/src/it/java/org/graalvm/tests/integration/utils/Apps.java index eb88578..985d248 100755 --- a/testsuite/src/it/java/org/graalvm/tests/integration/utils/Apps.java +++ b/testsuite/src/it/java/org/graalvm/tests/integration/utils/Apps.java @@ -22,14 +22,11 @@ import org.apache.commons.lang3.StringUtils; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; -import java.util.Properties; import static org.graalvm.tests.integration.RuntimesSmokeTest.BASE_DIR; import static org.graalvm.tests.integration.utils.thresholds.Thresholds.parseProperties; @@ -186,7 +183,17 @@ public enum Apps { URLContent.NONE, WhitelistLogLines.MONITOR_OFFSET, BuildAndRunCmds.MONITOR_OFFSET_BUILDER_IMAGE, - ContainerNames.MONITOR_OFFSET_BUILDER_IMAGE); + ContainerNames.MONITOR_OFFSET_BUILDER_IMAGE), + FOR_SERIALIZATION("apps" + File.separator + "for-serialization", + URLContent.NONE, + WhitelistLogLines.FOR_SERIALIZATION, + BuildAndRunCmds.FOR_SERIALIZATION, + ContainerNames.NONE), + FOR_SERIALIZATION_BUILDER_IMAGE("apps" + File.separator + "for-serialization", + URLContent.NONE, + WhitelistLogLines.FOR_SERIALIZATION, + BuildAndRunCmds.FOR_SERIALIZATION_BUILDER_IMAGE, + ContainerNames.FOR_SERIALIZATION_BUILDER_IMAGE); public final String dir; public final URLContent urlContent; diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/utils/BuildAndRunCmds.java b/testsuite/src/it/java/org/graalvm/tests/integration/utils/BuildAndRunCmds.java index 838b721..d8623fe 100755 --- a/testsuite/src/it/java/org/graalvm/tests/integration/utils/BuildAndRunCmds.java +++ b/testsuite/src/it/java/org/graalvm/tests/integration/utils/BuildAndRunCmds.java @@ -410,6 +410,41 @@ public enum BuildAndRunCmds { "--name", ContainerNames.MONITOR_OFFSET_BUILDER_IMAGE.name, BUILDER_IMAGE, "-R:-InstallSegfaultHandler", "-march=native", "--gc=serial", "--no-fallback", "-jar", "target/monitor-field-offsets-nok.jar", "target/monitor-field-offsets-nok" }, + }), + FOR_SERIALIZATION(new String[][] { + new String[] { "mvn", "package" }, + new String[] { "java", "-agentlib:native-image-agent=config-output-dir=src/main/resources/META-INF/native-image", + "-jar", "target/for-serialization.jar" }, + new String[] { "native-image", "-ea", "-march=native", "--no-fallback", "--link-at-build-time", + "-H:ConfigurationFileDirectories=src/main/resources/META-INF/native-image", + "-jar", "target/for-serialization.jar", "target/for-serialization" }, + new String[] { "java", "-jar", "target/for-serialization.jar" }, + new String[] { IS_THIS_WINDOWS ? "target\\for-serialization.exe" : "./target/for-serialization" } + }), + FOR_SERIALIZATION_BUILDER_IMAGE(new String[][] { + // Maven build + new String[] { "mvn", "package" }, + // Collect agent info + new String[] { CONTAINER_RUNTIME, "run", IS_THIS_WINDOWS ? "" : "-u", IS_THIS_WINDOWS ? "" : getUnixUIDGID(), + "-t", "--entrypoint", "java", "-v", BASE_DIR + File.separator + "apps" + File.separator + "for-serialization:/project:z", + BUILDER_IMAGE, "-agentlib:native-image-agent=config-output-dir=src/main/resources/META-INF/native-image", + "-jar", "target/for-serialization.jar" }, + // Native image build + new String[] { CONTAINER_RUNTIME, "run", IS_THIS_WINDOWS ? "" : "-u", IS_THIS_WINDOWS ? "" : getUnixUIDGID(), + "-t", "-v", BASE_DIR + File.separator + "apps" + File.separator + "for-serialization:/project:z", + BUILDER_IMAGE, "-ea", "-march=native", "--no-fallback", "--link-at-build-time", + "-H:ConfigurationFileDirectories=src/main/resources/META-INF/native-image", + "-jar", "target/for-serialization.jar", "target/for-serialization" }, + // Build runtime container for native executable + new String[] { CONTAINER_RUNTIME, "build", "--network=host", "-t", ContainerNames.FOR_SERIALIZATION_BUILDER_IMAGE.name, "." }, + // Run Java, HotSpot version in container, uses our builder image + new String[] { CONTAINER_RUNTIME, "run", IS_THIS_WINDOWS ? "" : "-u", IS_THIS_WINDOWS ? "" : getUnixUIDGID(), + "-t", "--entrypoint", "java", "-v", BASE_DIR + File.separator + "apps" + File.separator + "for-serialization:/project:z", + BUILDER_IMAGE, "-jar", "target/for-serialization.jar" }, + // Run native executable in container, uses a plain UBI image + new String[] { CONTAINER_RUNTIME, "run", IS_THIS_WINDOWS ? "" : "-u", IS_THIS_WINDOWS ? "" : getUnixUIDGID(), + "-t", "-v", BASE_DIR + File.separator + "apps" + File.separator + "for-serialization:/work:z", + ContainerNames.FOR_SERIALIZATION_BUILDER_IMAGE.name, "target/for-serialization" }, }); public final String[][] cmds; diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/utils/ContainerNames.java b/testsuite/src/it/java/org/graalvm/tests/integration/utils/ContainerNames.java index 1373903..27b5bc4 100755 --- a/testsuite/src/it/java/org/graalvm/tests/integration/utils/ContainerNames.java +++ b/testsuite/src/it/java/org/graalvm/tests/integration/utils/ContainerNames.java @@ -33,6 +33,7 @@ public enum ContainerNames { JFR_PLAINTEXT_BUILDER_IMAGE("my-jfr-plaintext-runner"), HYPERFOIL("hyperfoil-container"), MONITOR_OFFSET_BUILDER_IMAGE("my-monitor-offset-runner"), + FOR_SERIALIZATION_BUILDER_IMAGE("my-for-serialization-runner"), NONE("NO_CONTAINER"); public final String name; diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/utils/WhitelistLogLines.java b/testsuite/src/it/java/org/graalvm/tests/integration/utils/WhitelistLogLines.java index e9afd84..7feaf7e 100755 --- a/testsuite/src/it/java/org/graalvm/tests/integration/utils/WhitelistLogLines.java +++ b/testsuite/src/it/java/org/graalvm/tests/integration/utils/WhitelistLogLines.java @@ -316,6 +316,14 @@ public Pattern[] get(boolean inContainer) { Pattern.compile(".*error report at:.*"), }; } + }, + FOR_SERIALIZATION { + @Override + public Pattern[] get(boolean inContainer) { + return new Pattern[]{ + Pattern.compile(".*sun.reflect.ReflectionFactory is internal proprietary API.*") + }; + } }; public abstract Pattern[] get(boolean inContainer); From 907c2ab5685b99a3a63861e5a5e0b8b3eff52907 Mon Sep 17 00:00:00 2001 From: Karm Michal Babacek Date: Fri, 13 Sep 2024 10:47:44 +0200 Subject: [PATCH 2/4] Fix min/minJDK Co-authored-by: Foivos --- .../java/org/graalvm/tests/integration/AppReproducersTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java b/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java index 81331c5..9fa9118 100644 --- a/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java +++ b/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java @@ -825,7 +825,7 @@ public void forSerializationContainerTest(TestInfo testInfo) throws IOException, } @Test - @IfMandrelVersion(minJDK = "23.1.5") + @IfMandrelVersion(min = "23.1.5") public void forSerializationTest(TestInfo testInfo) throws IOException, InterruptedException { forSerialization(testInfo, Apps.FOR_SERIALIZATION); } From 0732e2b878a68cbcf926d72dc42701af59332b5e Mon Sep 17 00:00:00 2001 From: Michal Karm Babacek Date: Mon, 16 Sep 2024 14:46:08 +0200 Subject: [PATCH 3/4] Versions exclusions --- .../tests/integration/AppReproducersTest.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java b/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java index 9fa9118..a461b19 100644 --- a/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java +++ b/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java @@ -819,14 +819,27 @@ public void monitorFieldOffset(TestInfo testInfo, Apps app) throws IOException, @Test @Tag("builder-image") - @IfMandrelVersion(min = "23.1.5", inContainer = true) - public void forSerializationContainerTest(TestInfo testInfo) throws IOException, InterruptedException { + @IfMandrelVersion(min = "23.1.5", maxJDK = "21.999", inContainer = true) + public void forSerializationContainer21Test(TestInfo testInfo) throws IOException, InterruptedException { forSerialization(testInfo, Apps.FOR_SERIALIZATION_BUILDER_IMAGE); } @Test - @IfMandrelVersion(min = "23.1.5") - public void forSerializationTest(TestInfo testInfo) throws IOException, InterruptedException { + @IfMandrelVersion(min = "23.1.5", maxJDK = "21.999") + public void forSerialization21Test(TestInfo testInfo) throws IOException, InterruptedException { + forSerialization(testInfo, Apps.FOR_SERIALIZATION); + } + + @Test + @Tag("builder-image") + @IfMandrelVersion(min = "23.1.5", minJDK = "23", inContainer = true) + public void forSerializationContainer23Test(TestInfo testInfo) throws IOException, InterruptedException { + forSerialization(testInfo, Apps.FOR_SERIALIZATION_BUILDER_IMAGE); + } + + @Test + @IfMandrelVersion(min = "23.1.5", minJDK = "23") + public void forSerialization23Test(TestInfo testInfo) throws IOException, InterruptedException { forSerialization(testInfo, Apps.FOR_SERIALIZATION); } From be6f2eeabceb6f32887f39fac6580aa2690946cc Mon Sep 17 00:00:00 2001 From: Michal Karm Babacek Date: Tue, 17 Sep 2024 12:51:55 +0200 Subject: [PATCH 4/4] Fixes version maxJDK --- .../org/graalvm/tests/integration/AppReproducersTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java b/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java index a461b19..1f6c0c9 100644 --- a/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java +++ b/testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java @@ -819,13 +819,13 @@ public void monitorFieldOffset(TestInfo testInfo, Apps app) throws IOException, @Test @Tag("builder-image") - @IfMandrelVersion(min = "23.1.5", maxJDK = "21.999", inContainer = true) + @IfMandrelVersion(min = "23.1.5", maxJDK = "21.0.999", inContainer = true) public void forSerializationContainer21Test(TestInfo testInfo) throws IOException, InterruptedException { forSerialization(testInfo, Apps.FOR_SERIALIZATION_BUILDER_IMAGE); } @Test - @IfMandrelVersion(min = "23.1.5", maxJDK = "21.999") + @IfMandrelVersion(min = "23.1.5", maxJDK = "21.0.999") public void forSerialization21Test(TestInfo testInfo) throws IOException, InterruptedException { forSerialization(testInfo, Apps.FOR_SERIALIZATION); }