Introduce JVMPortableRestoreMode

Previously, the CRIURestoreNonPortableMode was used to determine if a
checkpoint could be taken after restore. If CRIURestoreNonPortableMode
was enabled (default mode) only a single checkpoint is allowed and the
JVM doesn't need to generate portable code upon restore. This capability
also allows the JVM to use standard security providers since there is no
risk for that state to be serialized.

There is a need to separate out the portability capabilites and the
behavioural changes that CRIURestoreNonPortableMode offers. There are
cases where one wants portability as well as standard security
capabilites for debugging purposes.

This PR separates out the portability capabilites and the behavioural
aspects by providing an option that forces portability upon restore
regardless of what CRIURestoreNonPortableMode is set to.

In the future we can add another option that forces portability in
non-CRIU mode, it its desired.

Signed-off-by: Tobi Ajila <[email protected]>

runtime/compiler/compile/J9Compilation.cpp

@@ -857,8 +857,9 @@ bool
 J9::Compilation::compilePortableCode()
    {
    return (self()->fej9()->inSnapshotMode() ||
-             (self()->compileRelocatableCode() &&
-                self()->fej9()->isPortableSCCEnabled()));
+             self()->fej9()->isPortableRestoreModeEnabled() ||
+                (self()->compileRelocatableCode() &&
+                   self()->fej9()->isPortableSCCEnabled()));
    }
 
 

runtime/compiler/control/HookedByTheJit.cpp

@@ -1856,7 +1856,7 @@ static void jitHookPrepareRestore(J9HookInterface * * hookInterface, UDATA event
     * remove the portability restrictions on the target CPU (used
     * for JIT compiles) to allow optimal code generation
     */
-   if (!javaVM->internalVMFunctions->isCheckpointAllowed(vmThread))
+   if (!javaVM->internalVMFunctions->isJVMInPortableRestoreMode(vmThread))
       {
       TR::Compiler->target.cpu = TR::CPU::detect(TR::Compiler->omrPortLib);
       jitConfig->targetProcessor = TR::Compiler->target.cpu.getProcessorDescription();

runtime/compiler/control/J9Options.cpp

@@ -1489,7 +1489,7 @@ void J9::Options::preProcessMmf(J9JavaVM *vm, J9JITConfig *jitConfig)
 
    if (J9_ARE_ANY_BITS_SET(vm->extendedRuntimeFlags2, J9_EXTENDED_RUNTIME2_ENABLE_PORTABLE_SHARED_CACHE)
 #if defined(J9VM_OPT_CRIU_SUPPORT)
-       || vm->internalVMFunctions->isCheckpointAllowed(vmThread)
+       || vm->internalVMFunctions->isJVMInPortableRestoreMode(vmThread)
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
        )
       {
@@ -2368,7 +2368,7 @@ bool J9::Options::preProcessJitServer(J9JavaVM *vm, J9JITConfig *jitConfig)
             if (implicitClientMode && useJitServerExplicitlySpecified)
                {
                compInfo->setRemoteCompilationRequestedAtBootstrap(true);
-               if (!ifuncs->isNonPortableRestoreMode(currentThread))
+               if (ifuncs->isJVMInPortableRestoreMode(currentThread))
                    compInfo->setCanPerformRemoteCompilationInCRIUMode(true);
                }
 #endif

runtime/compiler/control/JITClientCompilationThread.cpp

@@ -524,9 +524,10 @@ handleServerMessage(JITServer::ClientStream *client, TR_J9VM *fe, JITServer::Mes
             vmInfo._aotHeader = *header;
             }
          vmInfo._inSnapshotMode = fe->inSnapshotMode();
+         vmInfo._isPortableRestoreMode = fe->isPortableRestoreModeEnabled();
          vmInfo._isSnapshotModeEnabled = fe->isSnapshotModeEnabled();
 #if defined(J9VM_OPT_CRIU_SUPPORT)
-         vmInfo._isNonPortableRestoreMode = javaVM->internalVMFunctions->isNonPortableRestoreMode(vmThread);
+         vmInfo._isNonPortableRestoreMode = !javaVM->internalVMFunctions->isJVMInPortableRestoreMode(vmThread);
 #else
          vmInfo._isNonPortableRestoreMode = false;
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */

runtime/compiler/control/rossa.cpp

@@ -1999,7 +1999,7 @@ aboutToBootstrap(J9JavaVM * javaVM, J9JITConfig * jitConfig)
     * is because, the restore run may not be on the same machine as the one that created
     * the snapshot; thus the JIT code must be portable.
     */
-   if (javaVM->internalVMFunctions->isCheckpointAllowed(curThread))
+   if (javaVM->internalVMFunctions->isJVMInPortableRestoreMode(curThread))
       {
       TR::Compiler->target.cpu = TR::CPU::detectRelocatable(TR::Compiler->omrPortLib);
       if (!J9_ARE_ANY_BITS_SET(javaVM->extendedRuntimeFlags2, J9_EXTENDED_RUNTIME2_ENABLE_PORTABLE_SHARED_CACHE))

runtime/compiler/env/VMJ9.cpp

@@ -9377,6 +9377,16 @@ TR_J9VMBase::inSnapshotMode()
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
    }
 
+bool
+TR_J9VMBase::isPortableRestoreModeEnabled()
+   {
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+   return getJ9JITConfig()->javaVM->internalVMFunctions->isJVMInPortableRestoreMode(vmThread());
+#else /* defined(J9VM_OPT_CRIU_SUPPORT) */
+   return false;
+#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
+   }
+
 bool
 TR_J9VMBase::isSnapshotModeEnabled()
    {

runtime/compiler/env/VMJ9.h

@@ -1398,6 +1398,13 @@ class TR_J9VMBase : public TR_FrontEnd
     */
    virtual bool inSnapshotMode();
 
+   /**
+    * \brief Answers whether the JIT should generate portable restore code.
+    *
+    * \return True if portable restore code should be generated, false otherwise.
+    */
+   virtual bool isPortableRestoreModeEnabled();
+
    /**
     * \brief Answers whether checkpoint and restore mode is enabled (but not necessarily
     * whether snapshots can be taken or if any restores have already occurred).

runtime/compiler/env/VMJ9Server.cpp

@@ -2481,6 +2481,14 @@ TR_J9ServerVM::isSnapshotModeEnabled()
    return vmInfo->_isSnapshotModeEnabled;
    }
 
+bool
+TR_J9ServerVM::isPortableRestoreModeEnabled()
+   {
+   JITServer::ServerStream *stream = _compInfoPT->getMethodBeingCompiled()->_stream;
+   auto *vmInfo = _compInfoPT->getClientData()->getOrCacheVMInfo(stream);
+   return vmInfo->_isPortableRestoreMode;
+   }
+
 bool
 TR_J9SharedCacheServerVM::isClassLibraryMethod(TR_OpaqueMethodBlock *method, bool vettedForAOT)
    {

runtime/compiler/env/VMJ9Server.hpp

@@ -247,6 +247,7 @@ class TR_J9ServerVM: public TR_J9VM
    virtual bool isMethodHandleExpectedType(TR::Compilation *comp, TR::KnownObjectTable::Index mhIndex, TR::KnownObjectTable::Index expectedTypeIndex) override;
    virtual bool inSnapshotMode() override;
    virtual bool isSnapshotModeEnabled() override;
+   virtual bool isPortableRestoreModeEnabled() override;
 
 private:
    bool instanceOfOrCheckCastHelper(J9Class *instanceClass, J9Class* castClass, bool cacheUpdate);

runtime/compiler/net/CommunicationStream.hpp

@@ -118,7 +118,7 @@ class CommunicationStream
    // likely to lose an increment when merging/rebasing/etc.
    //
    static const uint8_t MAJOR_NUMBER = 1;
-   static const uint16_t MINOR_NUMBER = 52; // ID: vKDVVxR2fImMeQKnbv6I
+   static const uint16_t MINOR_NUMBER = 53; // ID: 7dZlozupV5RwUvR62RqE
    static const uint8_t PATCH_NUMBER = 0;
    static uint32_t CONFIGURATION_FLAGS;
 

runtime/compiler/runtime/JITClientSession.hpp

@@ -326,6 +326,7 @@ class ClientSessionData
       // Do not protect them with #if defined(J9VM_OPT_CRIU_SUPPORT) because we want JITServer to be
       // able to handle all clients whether or not they have CRIU support enabled
       bool _inSnapshotMode;
+      bool _isPortableRestoreMode;
       bool _isSnapshotModeEnabled;
       bool _isNonPortableRestoreMode;
       }; // struct VMInfo

runtime/oti/j9nonbuilder.h

@@ -4199,6 +4199,7 @@ typedef struct J9DelayedLockingOpertionsRecord {
 #define J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE 0x4
 #define J9VM_CRIU_IS_JDWP_ENABLED 0x8
 #define J9VM_CRIU_IS_THROW_ON_DELAYED_CHECKPOINT_ENABLED 0x10
+#define J9VM_CRIU_IS_PORTABLE_JVM_RESTORE_MODE 0x20
 
 typedef struct J9CRIUCheckpointState {
 	U_32 flags;
@@ -5020,6 +5021,7 @@ typedef struct J9InternalVMFunctions {
 	BOOLEAN (*isCRIUSupportEnabled_VM)(struct J9JavaVM *vm);
 	BOOLEAN (*isCheckpointAllowed)(struct J9VMThread *currentThread);
 	BOOLEAN (*isNonPortableRestoreMode)(struct J9VMThread *currentThread);
+	BOOLEAN (*isJVMInPortableRestoreMode)(struct J9VMThread *currentThread);
 	BOOLEAN (*runInternalJVMCheckpointHooks)(struct J9VMThread *currentThread, const char **nlsMsgFormat);
 	BOOLEAN (*runInternalJVMRestoreHooks)(struct J9VMThread *currentThread, const char **nlsMsgFormat);
 	BOOLEAN (*runDelayedLockRelatedOperations)(struct J9VMThread *currentThread);

runtime/oti/jvminit.h

@@ -426,6 +426,8 @@ enum INIT_STAGE {
 #define VMOPT_XXDISABLECRIU "-XX:-EnableCRIUSupport"
 #define VMOPT_XXENABLECRIUNONPORTABLEMODE "-XX:+CRIURestoreNonPortableMode"
 #define VMOPT_XXDISABLECRIUNONPORTABLEMODE "-XX:-CRIURestoreNonPortableMode"
+#define VMOPT_XXENABLEJVMRESTOREPORTABLEMODE "-XX:+JVMPortableRestoreMode"
+#define VMOPT_XXDISABLEJVMRESTOREPORTABLEMODE "-XX:-JVMPortableRestoreMode"
 #define VMOPT_XSHARECLASSES_DISABLEONRESTORE "-Xshareclasses:disableOnRestore"
 #define VMOPT_XXENABLETHROWONDELAYECHECKPOINTOPERATION "-XX:+ThrowOnDelayedCheckpointOperation"
 #define VMOPT_XXDISABLETHROWONDELAYECHECKPOINTOPERATION "-XX:-ThrowOnDelayedCheckpointOperation"

runtime/oti/vm_api.h

@@ -541,6 +541,19 @@ isCheckpointAllowed(J9VMThread *currentThread);
 BOOLEAN
 isNonPortableRestoreMode(J9VMThread *currentThread);
 
+/**
+ * @brief Queries if portable restore mode (specified via
+ * -XX:+JVMPortableRestoreMode) is enabled. If so, the JVM
+ * will remain in portable mode after restore. However, this
+ * will have no impact on the JCL and taking another checkpoint
+ * will still not be permitted.
+ *
+ * @param currentThread vmthread token
+ * @return TRUE if enabled, FALSE otherwise
+ */
+BOOLEAN
+isJVMInPortableRestoreMode(J9VMThread *currentThread);
+
 /**
  * @brief JVM hooks to run before performing a JVM checkpoint
  *

runtime/vm/CRIUHelpers.cpp

@@ -134,6 +134,12 @@ isNonPortableRestoreMode(J9VMThread *currentThread)
 	return J9_ARE_ALL_BITS_SET(currentThread->javaVM->checkpointState.flags, J9VM_CRIU_IS_NON_PORTABLE_RESTORE_MODE);
 }
 
+BOOLEAN
+isJVMInPortableRestoreMode(J9VMThread *currentThread)
+{
+	return (!isNonPortableRestoreMode(currentThread) || J9_ARE_ALL_BITS_SET(currentThread->javaVM->checkpointState.flags, J9VM_CRIU_IS_PORTABLE_JVM_RESTORE_MODE)) && isCRIUSupportEnabled(currentThread);
+}
+
 /**
  * This adds an internal CRIU hook to trace all heap objects of instanceType and its subclasses if specified.
  *

runtime/vm/intfunc.c

@@ -411,6 +411,7 @@ J9InternalVMFunctions J9InternalFunctions = {
 	isCRIUSupportEnabled_VM,
 	isCheckpointAllowed,
 	isNonPortableRestoreMode,
+	isJVMInPortableRestoreMode,
 	runInternalJVMCheckpointHooks,
 	runInternalJVMRestoreHooks,
 	runDelayedLockRelatedOperations,

runtime/vm/jvminit.c

@@ -3893,6 +3893,16 @@ processVMArgsFromFirstToLast(J9JavaVM * vm)
 		}
 	}
 
+	{
+		IDATA enableJVMRestorePortableeMode = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXENABLEJVMRESTOREPORTABLEMODE, NULL);
+		IDATA disableJVMRestorePortableMode = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXDISABLEJVMRESTOREPORTABLEMODE, NULL);
+		if (enableJVMRestorePortableeMode > disableJVMRestorePortableMode) {
+			if (J9_ARE_ALL_BITS_SET(vm->checkpointState.flags, J9VM_CRIU_IS_CHECKPOINT_ENABLED)) {
+				vm->checkpointState.flags |= J9VM_CRIU_IS_PORTABLE_JVM_RESTORE_MODE;
+			}
+		}
+	}
+
 	{
 		IDATA enableThrowOnDelayedCheckpointOperation = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXENABLETHROWONDELAYECHECKPOINTOPERATION, NULL);
 		IDATA disableThrowOnDelayedCheckpointOperation = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXDISABLETHROWONDELAYECHECKPOINTOPERATION, NULL);

test/functional/cmdLineTests/criu/playlist.xml

@@ -91,6 +91,8 @@
 		<variations>
 			<variation>-Xjit -XX:+CRIURestoreNonPortableMode</variation>
 			<variation>-Xint -XX:+CRIURestoreNonPortableMode</variation>
+			<variation>-XX:+JVMPortableRestoreMode</variation>
+			<variation>-Xjit:count=0 -XX:+JVMPortableRestoreMode</variation>
 			<variation>-Xjit:count=0 -XX:+CRIURestoreNonPortableMode</variation>
 			<variation>-Xgcpolicy:optthruput</variation>
 			<variation>-Xgcpolicy:optavgpause</variation>
@@ -191,6 +193,7 @@
 			<variation>-Xjit</variation>
 			<variation>-Xjit:count=0</variation>
 			<variation>-Xjit:vlog=vlog</variation>
+			<variation>-XX:+JVMPortableRestoreMode</variation>
 		</variations>
 		<command>
 			if [ -x $(Q)$(TEST_JDK_BIN)$(D)jitserver$(Q) ]; \
@@ -352,6 +355,7 @@
 			<variation>-Xjit -XX:+CRIURestoreNonPortableMode</variation>
 			<variation>-Xint -XX:+CRIURestoreNonPortableMode</variation>
 			<variation>-Xjit:count=0 -XX:+CRIURestoreNonPortableMode</variation>
+			<variation>-XX:+JVMPortableRestoreMode</variation>
 		</variations>
 		<command>
 			$(JAVA_COMMAND) $(CMDLINETESTER_JVM_OPTIONS) -Xdump \

test/functional/cmdLineTests/criu/src/org/openj9/criu/CRIUSimpleTest.java

@@ -25,6 +25,11 @@
 import java.nio.file.Paths;
 import java.nio.file.Path;
 
+import javax.swing.SwingUtilities;
+import javax.swing.JFrame;
+import java.lang.management.*;
+
+
 public class CRIUSimpleTest {
 
 	public static void main(String args[]) {
@@ -36,12 +41,24 @@ public static void main(String args[]) {
 		}
 	}
 
+	private static void loadNewClasses() {
+		try {
+			SwingUtilities.isEventDispatchThread();
+			JFrame frame = new JFrame("Test code");
+		} catch (Throwable t) {
+			//ignore
+		}
+
+		ThreadMXBean mxb = ManagementFactory.getThreadMXBean();
+	}
+
 	public static void checkpoints(int num_checkpoints) {
 		Path path = Paths.get("cpData");
 		System.out.println("Total checkpoint(s) " + num_checkpoints + ":\nPre-checkpoint");
 		for (int cur_checkpint = 1; cur_checkpint <= num_checkpoints; ++cur_checkpint) {
 			CRIUTestUtils.checkPointJVM(path);
 			System.out.println("Post-checkpoint " + cur_checkpint);
+			loadNewClasses();
 		}
 	}
 }