diff --git a/llvm/include/llvm/CodeGen/StackMaps.h b/llvm/include/llvm/CodeGen/StackMaps.h index 7b55849994a1fee..952c2227cb4a5b4 100644 --- a/llvm/include/llvm/CodeGen/StackMaps.h +++ b/llvm/include/llvm/CodeGen/StackMaps.h @@ -29,6 +29,8 @@ class MCStreamer; class raw_ostream; class TargetRegisterInfo; +unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI); + /// MI-level stackmap operands. /// /// MI stackmap operations take the form: diff --git a/llvm/include/llvm/Support/Yk.h b/llvm/include/llvm/Support/Yk.h index 2af995d3632b532..2d8e95afb19bf7b 100644 --- a/llvm/include/llvm/Support/Yk.h +++ b/llvm/include/llvm/Support/Yk.h @@ -8,5 +8,6 @@ void initYkOptions(void); // YKFIXME: all of our command-line arguments should be collected here instead // of us randomly introducing `extern bool`s all over the place. extern bool YkOptNoneAfterIRPasses; +extern bool YkDontOptFuncABI; #endif diff --git a/llvm/include/llvm/Transforms/Yk/ControlPoint.h b/llvm/include/llvm/Transforms/Yk/ControlPoint.h index 98175cb05d4da18..ea67a0259483fed 100644 --- a/llvm/include/llvm/Transforms/Yk/ControlPoint.h +++ b/llvm/include/llvm/Transforms/Yk/ControlPoint.h @@ -14,6 +14,9 @@ // right instruction in AOT from where to continue. #define YK_RECONSTRUCT_FRAMES "__ykrt_reconstruct_frames" +// The name of the patchpoint intrinsic we use for the control point. +#define CP_PPNAME "llvm.experimental.patchpoint.void" + namespace llvm { ModulePass *createYkControlPointPass(); } // namespace llvm diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index ca071ba9a6b8dc5..17de43361a881c4 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1982,10 +1982,18 @@ void AsmPrinter::emitFunctionBody() { break; default: + // So that the ykpt decoder can work without disasembling instructions + // to find call-sites and sucessor blocks, we encode that info + // statically into the blockmap at AOT compile time. + // + // Some things that look like calls in IR don't actually emit a + // call into the binary. Namely a stackmap intrinsic. + // + // Note that patchpoint and statepoint intrinsics, although similar to + // the stackmap intrinsic, do actually emit a call in the binary, so we + // DO need to include those callsites. if (YkExtendedLLVMBBAddrMapSection && MI.isCall() && - (MI.getOpcode() != TargetOpcode::STACKMAP) && - (MI.getOpcode() != TargetOpcode::PATCHPOINT) && - (MI.getOpcode() != TargetOpcode::STATEPOINT)) { + (MI.getOpcode() != TargetOpcode::STACKMAP)) { // Record the address of the call instruction itself. MCSymbol *YkPreCallSym = MF->getContext().createTempSymbol("yk_precall", true); diff --git a/llvm/lib/CodeGen/StackMaps.cpp b/llvm/lib/CodeGen/StackMaps.cpp index 44a041d912dc743..625456e59d845d6 100644 --- a/llvm/lib/CodeGen/StackMaps.cpp +++ b/llvm/lib/CodeGen/StackMaps.cpp @@ -24,7 +24,6 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCObjectFileInfo.h" -#include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCStreamer.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -211,7 +210,7 @@ unsigned StackMaps::getNextMetaArgIdx(const MachineInstr *MI, unsigned CurIdx) { } /// Go up the super-register chain until we hit a valid dwarf register number. -static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) { +unsigned llvm::getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) { int RegNum; for (MCPhysReg SR : TRI->superregs_inclusive(Reg)) { RegNum = TRI->getDwarfRegNum(SR, false); diff --git a/llvm/lib/CodeGen/Yk/FixStackmapsSpillReloads.cpp b/llvm/lib/CodeGen/Yk/FixStackmapsSpillReloads.cpp index 2ab7993f565245b..5840656079b5aa5 100644 --- a/llvm/lib/CodeGen/Yk/FixStackmapsSpillReloads.cpp +++ b/llvm/lib/CodeGen/Yk/FixStackmapsSpillReloads.cpp @@ -89,8 +89,10 @@ INITIALIZE_PASS_BEGIN(FixStackmapsSpillReloads, DEBUG_TYPE, "Fixup Stackmap Spil INITIALIZE_PASS_END(FixStackmapsSpillReloads, DEBUG_TYPE, "Fixup Stackmap Spills", false, false) +const TargetRegisterInfo *TRI; bool FixStackmapsSpillReloads::runOnMachineFunction(MachineFunction &MF) { + TRI = MF.getSubtarget().getRegisterInfo(); bool Changed = false; const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); for (MachineBasicBlock &MBB : MF) { @@ -144,7 +146,7 @@ bool FixStackmapsSpillReloads::runOnMachineFunction(MachineFunction &MF) { MOI++; while (MOI != MI.operands_end()) { if (MOI->isReg()) { - Register Reg = MOI->getReg(); + unsigned int Reg = getDwarfRegNum(MOI->getReg(), TRI); // Check if the register operand in the stackmap is a restored // spill. // Since implicit operands are ignored by stackmaps (they are not @@ -240,13 +242,15 @@ bool FixStackmapsSpillReloads::runOnMachineFunction(MachineFunction &MF) { if (TII->isCopyInstr(MI) || TII->isLoadFromStackSlotPostFE(MI, FI)) { // FIXME: Can there be multiple spill reloads here? Then this would // need to be a loop. - if (TII->isCopyInstr(MI) && Spills.count(MI.getOperand(1).getReg())) { + unsigned int Op0 = getDwarfRegNum(MI.getOperand(0).getReg(), TRI); + unsigned int Op1 = getDwarfRegNum(MI.getOperand(1).getReg(), TRI); + if (TII->isCopyInstr(MI) && Spills.count(Op1)) { // The source for this copy instruction is itself a spill reload. // So we need to lookup the spill for the source and apply this // instead. - Spills[MI.getOperand(0).getReg()] = Spills[MI.getOperand(1).getReg()]; + Spills[Op0] = Spills[Op1]; } else { - Spills[MI.getOperand(0).getReg()] = &MI; + Spills[Op0] = &MI; } } } diff --git a/llvm/lib/Support/Yk.cpp b/llvm/lib/Support/Yk.cpp index c2ade306bf10a79..e74d88821698107 100644 --- a/llvm/lib/Support/Yk.cpp +++ b/llvm/lib/Support/Yk.cpp @@ -96,6 +96,20 @@ struct CreateYkEmbedIRParser { } // namespace static ManagedStatic, CreateYkEmbedIRParser> YkEmbedIRParser; +bool YkDontOptFuncABI; +namespace { +struct CreateYkDontOptFuncABIParser { + static void *call() { + return new cl::opt( + "yk-dont-opt-func-abi", + cl::desc( + "Don't change the ABIs of functions during optimisation"), + cl::NotHidden, cl::location(YkDontOptFuncABI)); + } +}; +} // namespace +static ManagedStatic, CreateYkDontOptFuncABIParser> YkDontOptFuncABIParser; + void llvm::initYkOptions() { *YkExtendedLLVMBBAddrMapSectionParser; *YkStackMapOffsetFixParser; @@ -103,4 +117,5 @@ void llvm::initYkOptions() { *YkStackmapsSpillFixParser; *YkOptNoneAfterIRPassesParser; *YkEmbedIRParser; + *YkDontOptFuncABIParser; } diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index 47d0819c270616b..4d54df489bb611f 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -24,6 +24,7 @@ #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/CodeGen/MachineValueType.h" +#include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InlineAsm.h" @@ -62,22 +63,6 @@ X86AsmPrinter::X86AsmPrinter(TargetMachine &TM, const TargetInstrInfo *TII; const TargetRegisterInfo *TRI; -/// Go up the super-register chain until we hit a valid dwarf register number. -/// -/// (duplicated/adapted from StackMaps.cpp so as to not introduce extra link -/// dependencies) -static unsigned getDwarfRegNum(unsigned Reg) { - int RegNum; - for (MCPhysReg SR : TRI->superregs_inclusive(Reg)) { - RegNum = TRI->getDwarfRegNum(SR, false); - if (RegNum >= 0) - break; - } - - assert(RegNum >= 0 && "Invalid Dwarf register number."); - return (unsigned)RegNum; -} - /// Clear any mappings that map to the given register. void clearRhs(Register Reg, std::map> &SpillMap) { auto I = SpillMap.begin(); @@ -116,6 +101,13 @@ void processInstructions( continue; } + // Because a patchpoint already captures the live values at the exact + // moment we desire, there's no need to compute a spillmap for them nor do + // we have to "patch them up". We can just skip them. + if (Instr.getOpcode() == TargetOpcode::PATCHPOINT) { + continue; + } + // Copying a value from one register B to another A, creates a mapping from // A to B. If A is tracked by the stackmap, then B will also be tracked and // assigned the same value during deoptimisation. @@ -124,8 +116,8 @@ void processInstructions( const MachineOperand Rhs = Instr.getOperand(1); assert(Lhs.isReg() && "Is register."); assert(Rhs.isReg() && "Is register."); - auto LhsDwReg = getDwarfRegNum(Lhs.getReg()); - auto RhsDwReg = getDwarfRegNum(Rhs.getReg()); + auto LhsDwReg = getDwarfRegNum(Lhs.getReg(), TRI); + auto RhsDwReg = getDwarfRegNum(Rhs.getReg(), TRI); if (LhsDwReg == RhsDwReg) { // Moves like `mov rax, rax` are effectively a NOP for this analysis. continue; @@ -149,7 +141,7 @@ void processInstructions( const MachineOperand OffsetOp = Instr.getOperand(3); const MachineOperand MO = Instr.getOperand(5); assert(MO.isReg() && "Is register."); - const Register DwReg = getDwarfRegNum(MO.getReg()); + const Register DwReg = getDwarfRegNum(MO.getReg(), TRI); if (OffsetOp.isImm()) { const int64_t Offset = OffsetOp.getImm(); // We don't need to do `clearRhs(DwReg)` and reset the `SpillMap` entry @@ -168,7 +160,7 @@ void processInstructions( const MachineOperand OffsetOp = Instr.getOperand(4); const MachineOperand Lhs = Instr.getOperand(0); assert(Lhs.isReg() && "Is register."); - const Register DwReg = getDwarfRegNum(Lhs.getReg()); + const Register DwReg = getDwarfRegNum(Lhs.getReg(), TRI); if (OffsetOp.isImm()) { const int64_t Offset = OffsetOp.getImm(); clearRhs(DwReg, SpillMap); @@ -180,7 +172,7 @@ void processInstructions( // Any other assignments to tracked registers removes their mapping. for (const MachineOperand MO : Instr.defs()) { assert(MO.isReg() && "Is register."); - auto DwReg = getDwarfRegNum(MO.getReg()); + auto DwReg = getDwarfRegNum(MO.getReg(), TRI); SpillMap.erase(DwReg); clearRhs(DwReg, SpillMap); } @@ -190,7 +182,7 @@ void processInstructions( // be used. for (const MachineOperand MO : Instr.uses()) { if (MO.isReg() && MO.isKill()) { - auto DwReg = getDwarfRegNum(MO.getReg()); + auto DwReg = getDwarfRegNum(MO.getReg(), TRI); SpillMap.erase(DwReg); clearRhs(DwReg, SpillMap); } diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp index 56a52d13c20d0cc..f34a70d291a60b9 100644 --- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp +++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp @@ -58,6 +58,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Yk.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/Utils/CtorUtils.h" @@ -1986,7 +1987,9 @@ OptimizeFunctions(Module &M, } } - if (hasChangeableCC(&F) && !F.isVarArg() && !F.hasAddressTaken()) { + if (!YkDontOptFuncABI && hasChangeableCC(&F) && !F.isVarArg() + && !F.hasAddressTaken()) + { // If this function has a calling convention worth changing, is not a // varargs function, and is only called directly, promote it to use the // Fast calling convention. diff --git a/llvm/lib/Transforms/Yk/ControlPoint.cpp b/llvm/lib/Transforms/Yk/ControlPoint.cpp index a9c8b6ccbb76bf9..037003b5d7f02a1 100644 --- a/llvm/lib/Transforms/Yk/ControlPoint.cpp +++ b/llvm/lib/Transforms/Yk/ControlPoint.cpp @@ -1,4 +1,4 @@ -//===- ControlPoint.cpp - Synthesise the yk control point -----------------===// +//===- ControlPoint.cpp - Patch the yk control point -----------------===// // // This pass finds the user's call to the dummy control point and replaces it // with a call to a new control point that implements the necessary logic to @@ -19,23 +19,12 @@ // } // ``` // -// Into one that looks like this (note that this transformation happens at the -// IR level): +// Into one that looks like this: // // ``` -// // The YkCtrlPointStruct contains one member for each live LLVM variable -// // just before the call to the control point. -// struct YkCtrlPointStruct { -// size_t pc; -// } -// -// struct YkCtrlPointStruct cp_vars; // pc = 0; // while (...) { -// // Now we call the patched control point. -// cp_vars.pc = pc; -// __ykrt__control_point(mt, loc, &cp_vars); -// pc = cp_vars.pc; +// llvm.experimental.patchpoint.void(..., __ykrt_control_point, ...) // bc = program[pc]; // switch (bc) { // // bytecode handlers here. @@ -43,6 +32,9 @@ // } // ``` // +// A patchpoint is used to capture the locations of live variables immediately +// before a call to __ykrt_control_point. +// // Note that this transformation occurs at the LLVM IR level. The above example // is shown as C code for easy comprehension. @@ -58,6 +50,7 @@ #include "llvm/IR/Verifier.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" +#include "llvm/Transforms/Yk/LivenessAnalysis.h" #define DEBUG_TYPE "yk-control-point" @@ -68,6 +61,29 @@ #define YK_CONTROL_POINT_ARG_VARS_IDX 2 #define YK_CONTROL_POINT_NUM_ARGS 3 +// Stackmap ID zero is reserved for the control point. +// +// This will need to change when we support >1 control point. +const unsigned CPStackMapID = 0; + +// The number of shadow bytes required for the control point's patchpoint. +// +// This must be large enough to accommodate the call to patchpoint target +// function and if you use a too-big value LLVM will pad the space with NOP +// bytes. +// +// This early in the pipeline we have no idea how the backend will choose the +// encode this call, so for now we use the exact size of the observed +// instruction at the time of writing, as determined by disassembling the binary +// and eyeballing it. +// +// The good news is that LLVM will assert fail if you use a too small value. +#if defined(__x86_64__) || defined(_M_X64) +const unsigned CPShadow = 13; +#else +#error "unknown control point shadow size for this arch" +#endif + using namespace llvm; /// Find the call to the dummy control point that we want to patch. @@ -128,7 +144,7 @@ class YkControlPoint : public ModulePass { } // The old control point should be of the form: - // control_point(YkMT*, YkLocation*) + // yk_mt_control_point(YkMT*, YkLocation*) assert(OldCtrlPointCall->arg_size() == YK_OLD_CONTROL_POINT_NUM_ARGS); Type *YkMTTy = OldCtrlPointCall->getArgOperand(YK_CONTROL_POINT_ARG_MT_IDX)->getType(); @@ -136,26 +152,47 @@ class YkControlPoint : public ModulePass { OldCtrlPointCall->getArgOperand(YK_CONTROL_POINT_ARG_LOC_IDX) ->getType(); - // Create the new control point, which is of the form: - // void new_control_point(YkMT*, YkLocation*, i64) + // Create a call to the "new" (patched) control point, but do so via a + // patchpoint so that we can capture the live variables at exactly the + // moment before the call. Type *Int64Ty = Type::getInt64Ty(Context); FunctionType *FType = FunctionType::get(Type::getVoidTy(Context), {YkMTTy, YkLocTy, Int64Ty}, false); Function *NF = Function::Create(FType, GlobalVariable::ExternalLinkage, YK_NEW_CONTROL_POINT, M); - // At the top of the function, instantiate a `YkCtrlPointStruct` to pass in - // to the control point. We do so on the stack, so that we can pass the - // struct by pointer. IRBuilder<> Builder(OldCtrlPointCall); - // Insert call to the new control point. The last argument is the stackmap - // id belonging to the control point. This is temporarily set to INT_MAX - // and overwritten by the stackmap pass. - Builder.CreateCall( - NF, {OldCtrlPointCall->getArgOperand(YK_CONTROL_POINT_ARG_MT_IDX), - OldCtrlPointCall->getArgOperand(YK_CONTROL_POINT_ARG_LOC_IDX), - Builder.getInt64(UINT64_MAX)}); + const Intrinsic::ID SMFuncID = Function::lookupIntrinsicID(CP_PPNAME); + if (SMFuncID == Intrinsic::not_intrinsic) { + Context.emitError("can't find stackmap()"); + return false; + } + Function *SMFunc = Intrinsic::getDeclaration(&M, SMFuncID); + assert(SMFunc != nullptr); + + // Get live variables. + LivenessAnalysis LA(Caller); + auto Lives = LA.getLiveVarsBefore(OldCtrlPointCall); + + Value *SMID = ConstantInt::get(Type::getInt64Ty(Context), CPStackMapID); + Value *Shadow = ConstantInt::get(Type::getInt32Ty(Context), CPShadow); + std::vector Args = { + SMID, + Shadow, + NF, + ConstantInt::get(Type::getInt32Ty(Context), 3), + OldCtrlPointCall->getArgOperand(YK_CONTROL_POINT_ARG_MT_IDX), + OldCtrlPointCall->getArgOperand(YK_CONTROL_POINT_ARG_LOC_IDX), + SMID, + }; + + for (auto *Live : Lives) { + Args.push_back(Live); + } + + Builder.CreateCall(SMFunc->getFunctionType(), SMFunc, + ArrayRef(Args)); // Replace the call to the dummy control point. OldCtrlPointCall->eraseFromParent(); diff --git a/llvm/lib/Transforms/Yk/StackMaps.cpp b/llvm/lib/Transforms/Yk/StackMaps.cpp index 621ba66ab31e285..b88cc1f0af75f37 100644 --- a/llvm/lib/Transforms/Yk/StackMaps.cpp +++ b/llvm/lib/Transforms/Yk/StackMaps.cpp @@ -22,6 +22,14 @@ #define DEBUG_TYPE "yk-stackmaps" +// The first stackmap ID available for use by this pass. +// +// It's 1 because 0 is reserved for the control point. +// See llvm/lib/Transforms/Yk/ControlPoint.cpp +// +// This will have to change when we support >1 control point. +const int NonCPStackmapIDStart = 1; + using namespace llvm; namespace llvm { @@ -62,14 +70,19 @@ class YkStackmaps : public ModulePass { // can't tell if an indirect call is an intrinsic at compile time, // emit a stackmap in those cases too. - if (!CI.isIndirectCall() && - (CI.getCalledFunction()->isIntrinsic() || - (CI.getCalledFunction()->isDeclaration() && - (!CI.getCalledFunction()->getName().startswith( - "__yk_promote") && - CI.getCalledFunction()->getName() != YK_NEW_CONTROL_POINT)))) - continue; - + if (!CI.isIndirectCall()) { + if (CI.getCalledFunction()->isIntrinsic()) { + // This also skips the control point which is called via a + // patchpoint intrinsic, which already emits a stackmap. + continue; + } + + if (CI.getCalledFunction()->isDeclaration() && + !CI.getCalledFunction()->getName().startswith( + "__yk_promote")) { + continue; + } + } SMCalls.insert({&I, LA.getLiveVarsBefore(&I)}); } else if ((isa(I) && cast(I).isConditional()) || @@ -83,7 +96,7 @@ class YkStackmaps : public ModulePass { Function *SMFunc = Intrinsic::getDeclaration(&M, SMFuncID); assert(SMFunc != nullptr); - uint64_t Count = 0; + uint64_t Count = NonCPStackmapIDStart; Value *Shadow = ConstantInt::get(Type::getInt32Ty(Context), 0); for (auto It : SMCalls) { Instruction *I = cast(It.first); @@ -100,13 +113,6 @@ class YkStackmaps : public ModulePass { // the offset of the stackmap entry will record the instruction after // the call, which is where we want to continue after deoptimisation. Bldr.SetInsertPoint(I->getNextNode()); - CallInst &CI = cast(*I); - if (!CI.isIndirectCall() && - CI.getCalledFunction()->getName() == YK_NEW_CONTROL_POINT) { - // Update the stackmap id passed into the control point. - unsigned ArgSize = CI.arg_size(); - CI.setArgOperand(ArgSize - 1, SMID); - } } Bldr.CreateCall(SMFunc->getFunctionType(), SMFunc, ArrayRef(Args)); diff --git a/llvm/lib/YkIR/YkIRWriter.cpp b/llvm/lib/YkIR/YkIRWriter.cpp index e3b3e04abec325e..e1dd354900b9fe1 100644 --- a/llvm/lib/YkIR/YkIRWriter.cpp +++ b/llvm/lib/YkIR/YkIRWriter.cpp @@ -25,6 +25,13 @@ using namespace llvm; using namespace std; +// The argument index of the parameter of +// llvm.experimental.patchpoint.* +// +// We need this later to know where arguments to __ykrt_control_point stop, and +// where live variables to include in the stackmap entry start. +const int PPArgIdxNumTargetArgs = 3; + #include namespace { @@ -577,18 +584,29 @@ class YkIRWriter { void serialiseStackmapCall(CallInst *I, FuncLowerCtxt &FLCtxt) { assert(I); assert(I->getCalledFunction()->isIntrinsic()); - assert(I->getIntrinsicID() == Intrinsic::experimental_stackmap); + assert(I->getIntrinsicID() == Intrinsic::experimental_stackmap || + I->getIntrinsicID() == Intrinsic::experimental_patchpoint_void); // stackmap ID: Value *Op = I->getOperand(0); assert(isa(Op)); uint64_t SMId = (cast(Op))->getZExtValue(); OutStreamer.emitInt64(SMId); + int Skip = 0; + if (I->getIntrinsicID() == Intrinsic::experimental_stackmap) { + // Skip the following arguments: ID, shadow. + Skip = 2; + } else if (I->getIntrinsicID() == Intrinsic::experimental_patchpoint_void) { + // Skip the following arguments: ID, shadow, target, target arguments. + Skip = 4 + cast(I->getOperand(PPArgIdxNumTargetArgs)) + ->getZExtValue(); + } + // num_lives: - OutStreamer.emitInt32(I->arg_size() - 2); + OutStreamer.emitInt32(I->arg_size() - Skip); // lives: - for (unsigned OI = 2; OI < I->arg_size(); OI++) { + for (unsigned OI = Skip; OI < I->arg_size(); OI++) { serialiseOperand(I, FLCtxt, I->getOperand(OI)); } } @@ -759,13 +777,32 @@ class YkIRWriter { for (unsigned OI = 0; OI < I->arg_size(); OI++) { serialiseOperand(I, FLCtxt, I->getOperand(OI)); } - bool IsCtrlPointCall = - I->getCalledFunction()->getName() == YK_NEW_CONTROL_POINT; + bool IsCtrlPointCall = I->getCalledFunction()->getName() == CP_PPNAME; if (!I->getCalledFunction()->isDeclaration() || IsCtrlPointCall) { // The next instruction will be the stackmap entry // has_safepoint = 1: OutStreamer.emitInt8(1); - CallInst *SMI = dyn_cast(I->getNextNonDebugInstruction()); + CallInst *SMI = nullptr; + + // The control point is special. We use a patchpoint to perform the + // call, so the stackmap is associated with the patchpoint itself. + // + // We'd love to be able to do the same for ALL calls that need a + // stackmap, but patchpoints can only return void or i64, which isn't + // general enough for any given call we may encounter in the IR. + // + // For non-control-point calls, we instead place a stackmap instruction + // after the call and rely on a pass (FixStackmapsSpillReloads) to "patch + // up" the MIR later. This is necessary because we want the live + // locations at the point of the call, but when you place stackmap + // instruction after the call, you don't generally get that: LLVM often + // inserts instructions between the call and the stackmap instruction + // which is (for us, undesirably) reflected in the stackmap entry. + if (IsCtrlPointCall) { + SMI = dyn_cast(I); + } else { + SMI = dyn_cast(I->getNextNonDebugInstruction()); + } serialiseStackmapCall(SMI, FLCtxt); } else { // has_safepoint = 0: