diff --git a/runtime/compiler/control/CompilationRuntime.hpp b/runtime/compiler/control/CompilationRuntime.hpp index b927e9f6d93..1005883083a 100644 --- a/runtime/compiler/control/CompilationRuntime.hpp +++ b/runtime/compiler/control/CompilationRuntime.hpp @@ -452,7 +452,7 @@ class CompilationInfo static bool createCompilationInfo(J9JITConfig * jitConfig); static void freeCompilationInfo(J9JITConfig *jitConfig); static TR::CompilationInfo *get(J9JITConfig * = 0) { return _compilationRuntime; } - static bool shouldRetryCompilation(TR_MethodToBeCompiled *entry, TR::Compilation *comp); + static bool shouldRetryCompilation(J9VMThread *vmThread, TR_MethodToBeCompiled *entry, TR::Compilation *comp); static bool shouldAbortCompilation(TR_MethodToBeCompiled *entry, TR::PersistentInfo *persistentInfo); static bool canRelocateMethod(TR::Compilation * comp); static int computeCompilationThreadPriority(J9JavaVM *vm); diff --git a/runtime/compiler/control/CompilationThread.cpp b/runtime/compiler/control/CompilationThread.cpp index 12112d980a3..f7c3498bc32 100644 --- a/runtime/compiler/control/CompilationThread.cpp +++ b/runtime/compiler/control/CompilationThread.cpp @@ -2229,7 +2229,7 @@ bool TR::CompilationInfo::shouldAbortCompilation(TR_MethodToBeCompiled *entry, T // This method has side-effects, It modifies the optimization plan and persistentMethodInfo // This method is executed with compilationMonitor in hand // -bool TR::CompilationInfo::shouldRetryCompilation(TR_MethodToBeCompiled *entry, TR::Compilation *comp) +bool TR::CompilationInfo::shouldRetryCompilation(J9VMThread *vmThread, TR_MethodToBeCompiled *entry, TR::Compilation *comp) { // The JITServer should not retry compilations on it's own, // it should let the client make that decision @@ -2237,6 +2237,7 @@ bool TR::CompilationInfo::shouldRetryCompilation(TR_MethodToBeCompiled *entry, T return false; bool tryCompilingAgain = false; + TR::CompilationInfo *compInfo = entry->_compInfoPT->getCompilationInfo(); TR::IlGeneratorMethodDetails & details = entry->getMethodDetails(); J9Method *method = details.getMethod(); @@ -2274,7 +2275,7 @@ bool TR::CompilationInfo::shouldRetryCompilation(TR_MethodToBeCompiled *entry, T #if defined(J9VM_OPT_JITSERVER) case compilationStreamFailure: // if -XX:+JITServerRequireServer is used, we would like the client to fail when server crashes - if (entry->_compInfoPT->getCompilationInfo()->getPersistentInfo()->getRequireJITServer()) + if (compInfo->getPersistentInfo()->getRequireJITServer()) { TR_ASSERT_FATAL(false, "Option -XX:+JITServerRequireServer is used, terminate the JITClient due to unavailable JITServer."); } @@ -2459,10 +2460,36 @@ bool TR::CompilationInfo::shouldRetryCompilation(TR_MethodToBeCompiled *entry, T } }// if (entry->_compErrCode != compilationOK) + // If the JVM is shutting down, don't bother trying to compile again + if (compInfo->isInShutdownMode()) + tryCompilingAgain = false; + // don't carry the decision to generate AOT code to the next compilation // because it may no longer be the right thing to do at that point if (tryCompilingAgain) + { entry->_useAotCompilation = false; + } +#if defined(J9VM_OPT_CRIU_SUPPORT) + else if (entry->_compErrCode != compilationOK) + { + J9JavaVM *javaVM = compInfo->getJITConfig()->javaVM; + if (javaVM->internalVMFunctions->isDebugOnRestoreEnabled(vmThread) + && javaVM->internalVMFunctions->isCheckpointAllowed(vmThread) + && !compInfo->getCRRuntime()->isCheckpointInProgress() + && !compInfo->isInShutdownMode()) + { + if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestoreDetails)) + TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "Pushing failed compilation %p", entry->getMethodDetails().getMethod()); + + TR_FilterBST *filter = NULL; + OMR::CriticalSection pushFailedComp(compInfo->getCRRuntime()->getCRRuntimeMonitor()); + if (comp && entry->_compInfoPT->methodCanBeCompiled(comp->trMemory(), comp->fej9(), comp->getMethodBeingCompiled(), filter)) + compInfo->getCRRuntime()->pushFailedCompilation(entry->getMethodDetails().getMethod()); + } + } +#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */ + return tryCompilingAgain; } @@ -2520,6 +2547,11 @@ void TR::CompilationInfo::purgeMethodQueue(TR_CompilationErrorCode errorCode) getLowPriorityCompQueue().purgeLPQ(); // and from JProfiling queue getJProfilingCompQueue().purge(); + +#if defined(J9VM_OPT_CRIU_SUPPORT) + // and from the memoized compilations lists + getCRRuntime()->purgeMemoizedCompilations(); +#endif } void TR::CompilationInfoPerThread::suspendCompilationThread() @@ -6850,7 +6882,7 @@ TR::CompilationInfoPerThreadBase::installAotCachedMethod( { entry->_compErrCode = returnCode; entry->setAotCodeToBeRelocated(NULL); // reset if relocation failed - entry->_tryCompilingAgain = _compInfo.shouldRetryCompilation(entry, compiler); // this will set entry->_doNotUseAotCodeFromSharedCache = true; + entry->_tryCompilingAgain = _compInfo.shouldRetryCompilation(vmThread, entry, compiler); // this will set entry->_doNotUseAotCodeFromSharedCache = true; // Feature [Defect 172216/180384]: Implement hints for failed AOT validations. The idea is that the failed validations are due to the // fact that AOT methods are loaded at a lower count than when they were compiled so the JVM is given less opportunity to resolve things. @@ -7938,7 +7970,7 @@ TR::CompilationInfoPerThreadBase::postCompilationTasks(J9VMThread * vmThread, { metaData = 0; } - else if (TR::CompilationInfo::shouldRetryCompilation(entry, _compiler)) + else if (TR::CompilationInfo::shouldRetryCompilation(vmThread, entry, _compiler)) { startPC = entry->_oldStartPC; // startPC == oldStartPC means compilation failure entry->_tryCompilingAgain = true; @@ -10530,6 +10562,26 @@ TR::CompilationInfo::compilationEnd(J9VMThread * vmThread, TR::IlGeneratorMethod else { jitMethodTranslated(vmThread, method, startPC); +#if defined(J9VM_OPT_CRIU_SUPPORT) + if (jitConfig->javaVM->internalVMFunctions->isCheckpointAllowed(vmThread) + && jitConfig->javaVM->internalVMFunctions->isDebugOnRestoreEnabled(vmThread)) + { + if (comp->getRecompilationInfo() && comp->getRecompilationInfo()->getJittedBodyInfo()) + { + if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestoreDetails)) + TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "Will force %p to be recompiled post-restore", method); + + OMR::CriticalSection pushForcedRecomp(compInfo->getCRRuntime()->getCRRuntimeMonitor()); + compInfo->getCRRuntime()->pushForcedRecompilation(method); + } + else + { + if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestoreDetails)) + TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "Cannot force %p to be recompiled post-restore because the bodyInfo does not exist", method); + } + } +#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */ + } } } @@ -10871,6 +10923,8 @@ void TR::CompilationInfoPerThreadBase::logCompilationSuccess( case TR_PersistentMethodInfo::RecompDueToEdo: catchBlockCounter = bodyInfo->getMethodInfo()->getCatchBlockCounter(); recompReason = 'E'; break; + case TR_PersistentMethodInfo::RecompDueToCRIU: + recompReason = 'F'; break; } // end switch bodyInfo->getMethodInfo()->setReasonForRecompilation(0); // reset the flags } diff --git a/runtime/compiler/control/HookedByTheJit.cpp b/runtime/compiler/control/HookedByTheJit.cpp index 6073a247079..ab4ddf42fcf 100644 --- a/runtime/compiler/control/HookedByTheJit.cpp +++ b/runtime/compiler/control/HookedByTheJit.cpp @@ -1369,7 +1369,11 @@ static void jitMethodSampleInterrupt(J9VMThread* vmThread, IDATA handlerKey, voi * optimization that statistically should be useful. */ && !compInfo->getCRRuntime()->shouldSuspendThreadsForCheckpoint() -#endif + + /* Don't sample methods for recompilation pre-checkpoint if Debug On Restore is enabled */ + && (!jitConfig->javaVM->internalVMFunctions->isCheckpointAllowed(vmThread) + || !jitConfig->javaVM->internalVMFunctions->isDebugOnRestoreEnabled(vmThread)) +#endif /* defined(J9VM_OPT_CRIU_SUPPORT) */ && !compInfo->getPersistentInfo()->getDisableFurtherCompilation()) { static char *enableDebugDLT = feGetEnv("TR_DebugDLT"); @@ -1424,6 +1428,10 @@ static void jitMethodSampleInterrupt(J9VMThread* vmThread, IDATA handlerKey, voi * optimization that statistically should be useful. */ && !compInfo->getCRRuntime()->shouldSuspendThreadsForCheckpoint() + + /* Don't sample methods for recompilation pre-checkpoint if Debug On Restore is enabled */ + && (!jitConfig->javaVM->internalVMFunctions->isCheckpointAllowed(vmThread) + || !jitConfig->javaVM->internalVMFunctions->isDebugOnRestoreEnabled(vmThread)) #endif && !compInfo->getPersistentInfo()->getDisableFurtherCompilation()) { @@ -1895,6 +1903,7 @@ static void jitHookClassesUnload(J9HookInterface * * hookInterface, UDATA eventN J9VMThread * vmThread = unloadedEvent->currentThread; J9JITConfig * jitConfig = vmThread->javaVM->jitConfig; + TR_J9VMBase * vmj9 = TR_J9VMBase::get(jitConfig, vmThread); TR::CompilationInfo * compInfo = TR::CompilationInfo::get(jitConfig); TR::PersistentInfo * persistentInfo = compInfo->getPersistentInfo(); @@ -2055,7 +2064,9 @@ static void jitHookAnonClassesUnload(J9HookInterface * * hookInterface, UDATA ev } J9JITConfig *jitConfig = vmThread->javaVM->jitConfig; + TR_J9VMBase * vmj9 = TR_J9VMBase::get(jitConfig, vmThread); TR::CompilationInfo * compInfo = TR::CompilationInfo::get(jitConfig); + #if defined(J9VM_JIT_DYNAMIC_LOOP_TRANSFER) compInfo->cleanDLTRecordOnUnload(); if (compInfo->getDLT_HT()) @@ -2068,7 +2079,6 @@ static void jitHookAnonClassesUnload(J9HookInterface * * hookInterface, UDATA ev #if defined(J9VM_INTERP_PROFILING_BYTECODES) if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableIProfilerThread)) { - TR_J9VMBase * vmj9 = (TR_J9VMBase *)(TR_J9VMBase::get(jitConfig, vmThread)); TR_IProfiler *iProfiler = vmj9->getIProfiler(); if (iProfiler) // even if Iprofiler is disabled, there might be some buffers in the queue { // which need to be invalidated @@ -2085,6 +2095,11 @@ static void jitHookAnonClassesUnload(J9HookInterface * * hookInterface, UDATA ev for (J9Class* j9clazz = unloadedEvent->anonymousClassesToUnload; j9clazz; j9clazz = j9clazz->gcLink) { cgOnClassUnloading(j9clazz); + +#if defined(J9VM_OPT_CRIU_SUPPORT) + compInfo->getCRRuntime()->removeMethodsFromMemoizedCompilations(j9clazz); +#endif // defined(J9VM_OPT_CRIU_SUPPORT) + j9clazz->classLoader = NULL; } } @@ -2134,6 +2149,9 @@ static void jitHookClassUnload(J9HookInterface * * hookInterface, UDATA eventNum // remove from compilation request queue any methods that belong to this class fej9->acquireCompilationLock(); fej9->invalidateCompilationRequestsForUnloadedMethods(clazz, false); +#if defined(J9VM_OPT_CRIU_SUPPORT) + compInfo->getCRRuntime()->removeMethodsFromMemoizedCompilations(j9clazz); +#endif // defined(J9VM_OPT_CRIU_SUPPORT) fej9->releaseCompilationLock(); J9Method * resolvedMethods = (J9Method *) fej9->getMethods((TR_OpaqueClassBlock*)j9clazz); @@ -2412,6 +2430,10 @@ void jitClassesRedefined(J9VMThread * currentThread, UDATA classCount, J9JITRede TR_ASSERT(0,"JIT HCR should make all methods recompilable, so startPC=%p should have a persistentBodyInfo", startPC); } } + +#if defined(J9VM_OPT_CRIU_SUPPORT) + compInfo->getCRRuntime()->removeMethodsFromMemoizedCompilations(staleMethod); +#endif // defined(J9VM_OPT_CRIU_SUPPORT) } classPair = (J9JITRedefinedClass *) ((char *) classPair->methodList + (classPair->methodCount * sizeof(struct J9JITMethodEquivalence))); } @@ -2424,8 +2446,13 @@ void jitClassesRedefined(J9VMThread * currentThread, UDATA classCount, J9JITRede reportHookDetail(currentThread, "jitClassesRedefined", " Invalidate all compilation requests"); fe->invalidateCompilationRequestsForUnloadedMethods(NULL, true); - //clean up the trampolines + // clean up the trampolines TR::CodeCacheManager::instance()->onFSDDecompile(); + +#if defined(J9VM_OPT_CRIU_SUPPORT) + // purge memoized compilations + compInfo->getCRRuntime()->purgeMemoizedCompilations(); +#endif } fe->releaseCompilationLock(); @@ -6204,7 +6231,11 @@ static int32_t J9THREAD_PROC samplerThreadProc(void * entryarg) CompilationDensity compDensity; #endif /* defined(J9VM_OPT_JITSERVER) */ - TR_J9VMBase *fe = TR_J9VMBase::get(jitConfig, 0); +#if defined(J9VM_OPT_CRIU_SUPPORT) + bool forcedRecompilations = false; +#endif + + TR_J9VMBase *fe = TR_J9VMBase::get(jitConfig, samplerThread); j9thread_set_name(j9thread_self(), "JIT Sampler"); while (!shutdownSamplerThread) @@ -6218,7 +6249,8 @@ static int32_t J9THREAD_PROC samplerThreadProc(void * entryarg) crtTime += samplingPeriod; #if defined(J9VM_OPT_CRIU_SUPPORT) - if (vm->internalVMFunctions->isCheckpointAllowed(samplerThread) + if (vm->internalVMFunctions->isCheckpointAllowed(samplerThread)) + { /* It's ok to not acquire the comp monitor here. Even if at this * point a checkpoint isn't in progress but later it is, the * checkpoint will not complete until the sampler thread suspends @@ -6230,9 +6262,16 @@ static int32_t J9THREAD_PROC samplerThreadProc(void * entryarg) * isn't true (e.g., due to shutdown), then the sampler thread will * not suspend itself. */ - && compInfo->getCRRuntime()->shouldSuspendThreadsForCheckpoint()) + if (compInfo->getCRRuntime()->shouldSuspendThreadsForCheckpoint()) + suspendSamplerThreadForCheckpoint(samplerThread,jitConfig, compInfo); + } + else if (vm->internalVMFunctions->isDebugOnRestoreEnabled(samplerThread)) { - suspendSamplerThreadForCheckpoint(samplerThread,jitConfig, compInfo); + if (!forcedRecompilations && jitConfig->javaVM->phase == J9VM_PHASE_NOT_STARTUP) + { + forcedRecompilations = true; + compInfo->getCRRuntime()->recompileMethodsCompiledPreCheckpoint(); + } } #endif // #if defined(J9VM_OPT_CRIU_SUPPORT) diff --git a/runtime/compiler/control/J9CompilationStrategy.cpp b/runtime/compiler/control/J9CompilationStrategy.cpp index 44c450e3eb8..403b5e37b93 100644 --- a/runtime/compiler/control/J9CompilationStrategy.cpp +++ b/runtime/compiler/control/J9CompilationStrategy.cpp @@ -198,6 +198,13 @@ TR_OptimizationPlan *J9::CompilationStrategy::processEvent(TR_MethodEvent *event *newPlanCreated = true; } break; + case TR_MethodEvent::ForcedRecompilationPostRestore: + { + hotnessLevel = warm; + plan = TR_OptimizationPlan::alloc(hotnessLevel); + *newPlanCreated = true; + } + break; default: TR_ASSERT(0, "Bad event type %d", event->_eventType); } diff --git a/runtime/compiler/control/J9CompilationStrategy.hpp b/runtime/compiler/control/J9CompilationStrategy.hpp index 05163716da9..444aaf08ec7 100644 --- a/runtime/compiler/control/J9CompilationStrategy.hpp +++ b/runtime/compiler/control/J9CompilationStrategy.hpp @@ -59,6 +59,7 @@ class TR_MethodEvent JitCompilationInducedByDLT, HWPRecompilationTrigger, CompilationBeforeCheckpoint, + ForcedRecompilationPostRestore, NumEvents // must be the last one }; int32_t _eventType; diff --git a/runtime/compiler/control/OptionsPostRestore.cpp b/runtime/compiler/control/OptionsPostRestore.cpp index c9413ad5a0c..e63263929fe 100644 --- a/runtime/compiler/control/OptionsPostRestore.cpp +++ b/runtime/compiler/control/OptionsPostRestore.cpp @@ -502,6 +502,8 @@ J9::OptionsPostRestore::invalidateCompiledMethodsIfNeeded(bool invalidateAll) shouldInvalidateCompiledMethod(method, fej9, compilationFiltersExist)) { invalidateCompiledMethod(method, fej9); + if (!invalidateAll) + _compInfo->getCRRuntime()->removeMethodsFromMemoizedCompilations(method); } } } @@ -510,6 +512,9 @@ J9::OptionsPostRestore::invalidateCompiledMethodsIfNeeded(bool invalidateAll) } javaVM->internalVMFunctions->allClassesEndDo(&classWalkState); + if (invalidateAll) + _compInfo->getCRRuntime()->purgeMemoizedCompilations(); + j9nls_printf(PORTLIB, (UDATA) J9NLS_WARNING, J9NLS_JIT_CHECKPOINT_RESTORE_CODE_INVALIDATED); } } diff --git a/runtime/compiler/control/RecompilationInfo.hpp b/runtime/compiler/control/RecompilationInfo.hpp index c36d1ac07fb..cdf9a0965c6 100644 --- a/runtime/compiler/control/RecompilationInfo.hpp +++ b/runtime/compiler/control/RecompilationInfo.hpp @@ -229,6 +229,8 @@ class TR_PersistentMethodInfo RecompDueToRI = 0x000A0000, RecompDueToJProfiling = 0x000B0000, RecompDueToInlinedMethodRedefinition = 0x000C0000, + RecompDueToCRIU = 0x000D0000, + // NOTE: recompilations due to EDO decrementation cannot be tracked precisely // because they are triggered from a snippet (must change the code for snippet) // Also, the recompilations after a profiling step cannot be marked as such. diff --git a/runtime/compiler/runtime/CRRuntime.cpp b/runtime/compiler/runtime/CRRuntime.cpp index 20f4f35f85b..b5cb81c525e 100644 --- a/runtime/compiler/runtime/CRRuntime.cpp +++ b/runtime/compiler/runtime/CRRuntime.cpp @@ -24,22 +24,32 @@ #include "j9nonbuilder.h" #include "j9thread.h" +#include "control/CompilationController.hpp" #include "control/CompilationRuntime.hpp" +#include "control/CompilationStrategy.hpp" #include "control/CompileBeforeCheckpoint.hpp" #include "control/Options.hpp" #include "control/OptionsPostRestore.hpp" -#include "env/RawAllocator.hpp" +#include "control/Recompilation.hpp" +#include "control/RecompilationInfo.hpp" #include "env/J9SegmentAllocator.hpp" +#include "env/RawAllocator.hpp" #include "env/SystemSegmentProvider.hpp" #include "env/VerboseLog.hpp" +#include "env/VMAccessCriticalSection.hpp" +#include "infra/Assert.hpp" #include "infra/CriticalSection.hpp" #include "infra/Monitor.hpp" #include "runtime/CRRuntime.hpp" #include "runtime/J9VMAccess.hpp" + #if defined(J9VM_OPT_JITSERVER) #include "net/ClientStream.hpp" #endif +template void TR::CRRuntime::removeMethodsFromMemoizedCompilations(J9Class *entryToRemove); +template void TR::CRRuntime::removeMethodsFromMemoizedCompilations(J9Method *entryToRemove); + class ReleaseVMAccessAndAcquireMonitor { public: @@ -75,7 +85,9 @@ TR::CRRuntime::CRRuntime(J9JITConfig *jitConfig, TR::CompilationInfo *compInfo) _crRuntimeThread(NULL), _crRuntimeOSThread(NULL), _crRuntimeThreadLifetimeState(TR_CRRuntimeThreadLifetimeStates::CR_THR_NOT_CREATED), - _checkpointStatus(TR_CheckpointStatus::NO_CHECKPOINT_IN_PROGRESS) + _checkpointStatus(TR_CheckpointStatus::NO_CHECKPOINT_IN_PROGRESS), + _failedComps(), + _forcedRecomps() { // TR::CompilationInfo is initialized in the JIT_INITIALIZED bootstrap // stage, whereas J9_EXTENDED_RUNTIME_METHOD_TRACE_ENABLED is set in the @@ -148,6 +160,210 @@ TR::CRRuntime::waitOnCRRuntimeMonitor() _crRuntimeMonitor->wait(); } +void +TR::CRRuntime::pushMemoizedCompilation(TR_MemoizedCompilations& list, J9Method *method) + { + auto newEntry = new (_compInfo->persistentMemory()) TR_MemoizedComp(method); + if (newEntry) + list.add(newEntry); + } + +J9Method * +TR::CRRuntime::popMemoizedCompilation(TR_MemoizedCompilations& list) + { + J9Method *method = NULL; + auto memComp = list.pop(); + if (memComp) + { + method = memComp->getMethod(); + jitPersistentFree(memComp); + } + return method; + } + +static bool shouldRemoveMemoizedCompilation(J9Method *j9methodInList, J9Class *j9class) + { + return J9_CLASS_FROM_METHOD(j9methodInList) == j9class; + } + +static bool shouldRemoveMemoizedCompilation(J9Method *j9methodInList, J9Method *j9method) + { + return j9methodInList == j9method; + } + +template +void +TR::CRRuntime::removeMemoizedCompilation(TR_MemoizedCompilations& list, T *entryToRemove) + { + if (!list.isEmpty()) + { + // Remove from front of list + auto curr = list.getFirst(); + while (curr) + { + J9Method *j9methodInList = curr->getMethod(); + + if (shouldRemoveMemoizedCompilation(j9methodInList, entryToRemove)) + jitPersistentFree(list.pop()); + else + break; + + curr = list.getFirst(); + } + + // Remove from middle of list + auto prev = curr; + curr = prev ? prev->getNext() : NULL; + while (curr) + { + J9Method *j9methodInList = curr->getMethod(); + + if (shouldRemoveMemoizedCompilation(j9methodInList, entryToRemove)) + { + list.removeAfter(prev, curr); + jitPersistentFree(curr); + } + else + { + prev = curr; + } + + curr = prev->getNext(); + } + } + } + +template +void +TR::CRRuntime::removeMethodsFromMemoizedCompilations(T *entryToRemove) + { + OMR::CriticalSection removeMemoizedCompilations(getCRRuntimeMonitor()); + removeMemoizedCompilation(_failedComps, entryToRemove); + removeMemoizedCompilation(_forcedRecomps, entryToRemove); + } + +void +TR::CRRuntime::purgeMemoizedCompilation(TR_MemoizedCompilations& list) + { + while (!list.isEmpty()) + { + jitPersistentFree(list.pop()); + } + } + +void +TR::CRRuntime::purgeMemoizedCompilations() + { + OMR::CriticalSection removeMemoizedCompilations(getCRRuntimeMonitor()); + purgeMemoizedCompilation(_failedComps); + purgeMemoizedCompilation(_forcedRecomps); + } + +void +TR::CRRuntime::triggerCompilationOfFailedCompilationsPreCheckpoint(J9VMThread *vmThread) + { + TR_J9VMBase *fe = TR_J9VMBase::get(getJITConfig(), vmThread); + + J9Method *method; + while ((method = popFailedCompilation())) + { + if (_compInfo->getJ9MethodVMExtra(method) == J9_JIT_NEVER_TRANSLATE) + { + if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestoreDetails)) + TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "%p Resetting %p", vmThread, method); + TR::CompilationInfo::setInvocationCount(method, 0); + } + else if (J9_ARE_ANY_BITS_SET(J9_ROM_METHOD_FROM_RAM_METHOD(method)->modifiers, J9AccNative)) + { + if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestoreDetails)) + TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "%p Requeuing %p", vmThread, method); + + TR_MethodEvent event; + event._eventType = TR_MethodEvent::InterpreterCounterTripped; + event._j9method = method; + event._oldStartPC = NULL; + event._vmThread = vmThread; + event._classNeedingThunk = 0; + bool newPlanCreated = false; + TR_OptimizationPlan *plan = TR::CompilationController::getCompilationStrategy()->processEvent(&event, &newPlanCreated); + // The plan needs to be created only when async compilation is possible + // Otherwise the compilation will be triggered on next invocation + if (plan) + { + bool queued = false; + + // Release the CR Runtime Monitor here as compileMethod will + // eventually acquire the Comp Monitor. + releaseCRRuntimeMonitor(); + + // scope for details + { + TR::VMAccessCriticalSection triggerFailedComps(fe); + + TR::IlGeneratorMethodDetails details(method); + _compInfo->compileMethod(vmThread, details, NULL, TR_maybe, NULL, &queued, plan); + } + + // Re-acquire the CR Runtime Monitor now that this thread does not + // have the Comp Monitor in hand. + acquireCRRuntimeMonitor(); + + if (!queued && newPlanCreated) + TR_OptimizationPlan::freeOptimizationPlan(plan); + } + } + } + } + +void +TR::CRRuntime::triggerRecompilationForPreCheckpointGeneratedFSDBodies(J9VMThread *vmThread) + { + TR_J9VMBase *fe = TR_J9VMBase::get(getJITConfig(), vmThread); + + J9Method *method; + while ((method = popForcedRecompilation())) + { + if (_compInfo->isCompiled(method)) + { + if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerboseCheckpointRestoreDetails)) + TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "%p Attempting to force %p for recompilation", vmThread, method); + + TR_MethodEvent event; + event._eventType = TR_MethodEvent::ForcedRecompilationPostRestore; + event._j9method = method; + event._oldStartPC = method->extra; + event._vmThread = vmThread; + event._classNeedingThunk = 0; + bool newPlanCreated = false; + TR_OptimizationPlan *plan = TR::CompilationController::getCompilationStrategy()->processEvent(&event, &newPlanCreated); + // the plan needs to be created only when async compilation is possible + // Otherwise the compilation will be triggered on next invocation + if (plan) + { + TR_PersistentJittedBodyInfo *bodyInfo = TR::Recompilation::getJittedBodyInfoFromPC(method->extra); + TR_PersistentMethodInfo *methodInfo = bodyInfo->getMethodInfo(); + methodInfo->setReasonForRecompilation(TR_PersistentMethodInfo::RecompDueToCRIU); + + bool queued = false; + + // Release the CR Runtime Monitor here as induceRecompilation will + // eventually acquire the Comp Monitor. + releaseCRRuntimeMonitor(); + + // Induce the recompilation + TR::Recompilation::induceRecompilation(fe, method->extra, &queued, plan); + + // Re-acquire the CR Runtime Monitor now that this thread does not + // have the Comp Monitor in hand. + acquireCRRuntimeMonitor(); + + if (!queued && newPlanCreated) + TR_OptimizationPlan::freeOptimizationPlan(plan); + } + } + } + } + /* IMPORTANT: There should be no return, or C++ exception thrown, after * releasing the Comp Monitor below until it is re-acquired. * The Comp Monitor may be acquired in an OMR::CriticalSection @@ -493,6 +709,20 @@ TR::CRRuntime::prepareForRestore() TR_VerboseLog::writeLineLocked(TR_Vlog_CHECKPOINT_RESTORE, "Ready for restore"); } +void +TR::CRRuntime::recompileMethodsCompiledPreCheckpoint() + { + if (!getCRRuntimeThread()) + return; + + OMR::CriticalSection recompFSDBodies(getCRRuntimeMonitor()); + if (getCRRuntimeThreadLifetimeState() == TR_CRRuntimeThreadLifetimeStates::CR_THR_INITIALIZED) + { + setCRRuntimeThreadLifetimeState(TR_CRRuntimeThreadLifetimeStates::CR_THR_TRIGGER_RECOMP); + getCRRuntimeMonitor()->notifyAll(); + } + } + void TR::CRRuntime::process() { @@ -504,11 +734,26 @@ TR::CRRuntime::process() waitOnCRRuntimeMonitor(); } - if (getCRRuntimeThreadLifetimeState() == TR_CRRuntimeThreadLifetimeStates::CR_THR_STOPPING) + auto state = getCRRuntimeThreadLifetimeState(); + if (state == TR_CRRuntimeThreadLifetimeStates::CR_THR_STOPPING) { releaseCRRuntimeMonitor(); break; } + else if (state == TR_CRRuntimeThreadLifetimeStates::CR_THR_TRIGGER_RECOMP) + { + triggerCompilationOfFailedCompilationsPreCheckpoint(getCRRuntimeThread()); + triggerRecompilationForPreCheckpointGeneratedFSDBodies(getCRRuntimeThread()); + + // Because the CR Runtime Monitor may have been released, only reset + // the state if the current state is still CR_THR_TRIGGER_RECOMP + if (getCRRuntimeThreadLifetimeState() == TR_CRRuntimeThreadLifetimeStates::CR_THR_TRIGGER_RECOMP) + setCRRuntimeThreadLifetimeState(TR_CRRuntimeThreadLifetimeStates::CR_THR_INITIALIZED); + } + else + { + TR_ASSERT_FATAL(false, "Invalid state %d\n", state); + } } while (true); } diff --git a/runtime/compiler/runtime/CRRuntime.hpp b/runtime/compiler/runtime/CRRuntime.hpp index 211823ee9bd..2d7b0d565f1 100644 --- a/runtime/compiler/runtime/CRRuntime.hpp +++ b/runtime/compiler/runtime/CRRuntime.hpp @@ -24,6 +24,7 @@ #include "j9.h" #include "env/TRMemory.hpp" +#include "infra/Link.hpp" extern "C" { struct J9JITConfig; @@ -56,6 +57,7 @@ class CRRuntime CR_THR_NOT_CREATED = 0, CR_THR_FAILED_TO_ATTACH, CR_THR_INITIALIZED, + CR_THR_TRIGGER_RECOMP, CR_THR_STOPPING, CR_THR_DESTROYED, CR_THR_LAST_STATE // must be the last one @@ -122,6 +124,30 @@ class CRRuntime void startCRRuntimeThread(J9JavaVM *javaVM); void stopCRRuntimeThread(); + /* The following methods should be only be invoked with the CR Runtime + * Monitor in hand. + */ + void pushFailedCompilation(J9Method *method) { pushMemoizedCompilation(_failedComps, method); } + J9Method * popFailedCompilation() { return popMemoizedCompilation(_failedComps); } + void pushForcedRecompilation(J9Method *method) { pushMemoizedCompilation(_forcedRecomps, method); } + J9Method * popForcedRecompilation() { return popMemoizedCompilation(_forcedRecomps); } + + /** + * @brief Remove appropriate methods from all of the memoized lists. This + * method acquires the CR Runtime Monitor. + * + * @param entryToRemove The entry that determines what should be removed from + * the various memoized lists. + */ + template + void removeMethodsFromMemoizedCompilations(T *entryToRemove); + + /** + * @brief Empty the various memoized lists. This method acquires the CR + * Runtime Monitor. + */ + void purgeMemoizedCompilations(); + /** * @brief Processing method that the CR Runtime Thread executes. */ @@ -144,8 +170,27 @@ class CRRuntime */ void prepareForRestore(); + /** + * @brief Notify the CR Runtime Thread to recompile methods that were + * memoized pre-checkpoint. + */ + void recompileMethodsCompiledPreCheckpoint(); + private: + class TR_MemoizedComp : public TR_Link0 + { + public: + TR_PERSISTENT_ALLOC(TR_MemoryBase::CompilationInfo); + + TR_MemoizedComp(J9Method *method) { _method = method; } + J9Method *getMethod() { return _method; } + + private: + J9Method *_method; + }; + typedef TR_LinkHead0 TR_MemoizedCompilations; + /* These are private since this monitor should only be externally accessed * via the TR::CompilationInfo object. */ @@ -217,6 +262,67 @@ class CRRuntime */ void resetStartTime(); + /** + * @brief Helper method to push a J9Method onto the front of list that is + * used to memoize a future compilation of said J9Method. This method + * should only be called with the Comp Monitor in hand. + * + * @param list A reference to the TR_MemoizedCompilations list + * @param method The method whose compilation should be memoized + */ + void pushMemoizedCompilation(TR_MemoizedCompilations& list, J9Method *method); + + /** + * @brief Helper method to pop a J9Method from a list that is used to memoize + * a future compilation of said J9Method; this action will remove the + * J9Method from the list. This method should only be called with the + * Comp Monitor in hand. + * + * @param list A reference to the TR_MemoizedCompilations list + * + * @return the J9Method at the front of the list; NULL if the list is empty. + */ + J9Method * popMemoizedCompilation(TR_MemoizedCompilations& list); + + /** + * @brief Helper method to remove a J9Method from a list that is used to + * memoize a future compilation of said J9Method. This method should + * only be called with the Comp Monitor in hand. + * + * @param list A reference to the TR_MemoizedCompilations list + * @param clazz The class of the methods to be removed from the list + */ + template + void removeMemoizedCompilation(TR_MemoizedCompilations& list, T *entryToRemove); + + /** + * @brief Empty the entries in the list that is passed to this method. + * + * @param list A reference to the TR_MemoizedCompilations list + */ + void purgeMemoizedCompilation(TR_MemoizedCompilations& list); + + /** + * @brief Trigger compilation of any compilations that failed pre-checkpoint; + * specifically, any compilations added to the _failedComps list. + * + * This method is called with the CR Runtime Monitor in hand. + * + * @param vmThread the J9VMThread + */ + void triggerCompilationOfFailedCompilationsPreCheckpoint(J9VMThread *vmThread); + + /** + * @brief Trigger recompilation of the compilations that were generated with + * FSD enabled pre-checkpoint; specifically, any compilations added to + * the _forcedRecomps list. + * + * This method is called with the CR Runtime Monitor in hand. + * + * @param vmThread the J9VMThread + */ + void triggerRecompilationForPreCheckpointGeneratedFSDBodies(J9VMThread *vmThread); + J9JITConfig *_jitConfig; TR::CompilationInfo *_compInfo; @@ -230,6 +336,9 @@ class CRRuntime TR_CRRuntimeThreadLifetimeStates _crRuntimeThreadLifetimeState; TR_CheckpointStatus _checkpointStatus; + TR_MemoizedCompilations _failedComps; + TR_MemoizedCompilations _forcedRecomps; + bool _vmMethodTraceEnabled; bool _vmExceptionEventsHooked;