[SystemZ][XRay] Implement XRay instrumentation for SystemZ (llvm#113253)

Expands pseudo instructions PATCHABLE_FUNCTION_ENTER and PATCHABLE_RET
into a small instruction sequence which calls into the XRay library.

llvm/lib/CodeGen/XRayInstrumentation.cpp

@@ -241,7 +241,8 @@ bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) {
       prependRetWithPatchableExit(MF, TII, op);
       break;
     }
-    case Triple::ArchType::ppc64le: {
+    case Triple::ArchType::ppc64le:
+    case Triple::ArchType::systemz: {
       // PPC has conditional returns. Turn them into branch and plain returns.
       InstrumentationOptions op;
       op.HandleTailcall = false;

llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp

@@ -15,6 +15,7 @@
 #include "MCTargetDesc/SystemZGNUInstPrinter.h"
 #include "MCTargetDesc/SystemZHLASMInstPrinter.h"
 #include "MCTargetDesc/SystemZMCExpr.h"
+#include "MCTargetDesc/SystemZMCTargetDesc.h"
 #include "SystemZConstantPoolValue.h"
 #include "SystemZMCInstLower.h"
 #include "TargetInfo/SystemZTargetInfo.h"
@@ -662,6 +663,23 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) {
     LowerPATCHPOINT(*MI, Lower);
     return;
 
+  case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
+    LowerPATCHABLE_FUNCTION_ENTER(*MI, Lower);
+    return;
+
+  case TargetOpcode::PATCHABLE_RET:
+    LowerPATCHABLE_RET(*MI, Lower);
+    return;
+
+  case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
+    llvm_unreachable("PATCHABLE_FUNCTION_EXIT should never be emitted");
+
+  case TargetOpcode::PATCHABLE_TAIL_CALL:
+    // TODO: Define a trampoline `__xray_FunctionTailExit` and differentiate a
+    // normal function exit from a tail exit.
+    llvm_unreachable("Tail call is handled in the normal case. See comments "
+                     "around this assert.");
+
   case SystemZ::EXRL_Pseudo: {
     unsigned TargetInsOpc = MI->getOperand(0).getImm();
     Register LenMinus1Reg = MI->getOperand(1).getReg();
@@ -844,6 +862,84 @@ void SystemZAsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
                             getSubtargetInfo());
 }
 
+void SystemZAsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(
+    const MachineInstr &MI, SystemZMCInstLower &Lower) {
+  // .begin:
+  //   j .end    # -> stmg    %r2, %r15, 16(%r15)
+  //   nop
+  //   llilf   %2, FuncID
+  //   brasl   %r14, __xray_FunctionEntry@GOT
+  // .end:
+  //
+  // Update compiler-rt/lib/xray/xray_s390x.cpp accordingly when number
+  // of instructions change.
+  bool HasVectorFeature =
+      TM.getMCSubtargetInfo()->hasFeature(SystemZ::FeatureVector) &&
+      !TM.getMCSubtargetInfo()->hasFeature(SystemZ::FeatureSoftFloat);
+  MCSymbol *FuncEntry = OutContext.getOrCreateSymbol(
+      HasVectorFeature ? "__xray_FunctionEntryVec" : "__xray_FunctionEntry");
+  MCSymbol *BeginOfSled = OutContext.createTempSymbol("xray_sled_", true);
+  MCSymbol *EndOfSled = OutContext.createTempSymbol();
+  OutStreamer->emitLabel(BeginOfSled);
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(SystemZ::J)
+                     .addExpr(MCSymbolRefExpr::create(EndOfSled, OutContext)));
+  EmitNop(OutContext, *OutStreamer, 2, getSubtargetInfo());
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(SystemZ::LLILF).addReg(SystemZ::R2D).addImm(0));
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(SystemZ::BRASL)
+                     .addReg(SystemZ::R14D)
+                     .addExpr(MCSymbolRefExpr::create(
+                         FuncEntry, MCSymbolRefExpr::VK_PLT, OutContext)));
+  OutStreamer->emitLabel(EndOfSled);
+  recordSled(BeginOfSled, MI, SledKind::FUNCTION_ENTER, 2);
+}
+
+void SystemZAsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
+                                           SystemZMCInstLower &Lower) {
+  unsigned OpCode = MI.getOperand(0).getImm();
+  MCSymbol *FallthroughLabel = nullptr;
+  if (OpCode == SystemZ::CondReturn) {
+    FallthroughLabel = OutContext.createTempSymbol();
+    int64_t Cond0 = MI.getOperand(1).getImm();
+    int64_t Cond1 = MI.getOperand(2).getImm();
+    EmitToStreamer(*OutStreamer, MCInstBuilder(SystemZ::BRC)
+                                     .addImm(Cond0)
+                                     .addImm(Cond1 ^ Cond0)
+                                     .addExpr(MCSymbolRefExpr::create(
+                                         FallthroughLabel, OutContext)));
+  }
+  // .begin:
+  //   br %r14    # -> stmg    %r2, %r15, 24(%r15)
+  //   nop
+  //   nop
+  //   llilf   %2,FuncID
+  //   j       __xray_FunctionExit@GOT
+  //
+  // Update compiler-rt/lib/xray/xray_s390x.cpp accordingly when number
+  // of instructions change.
+  bool HasVectorFeature =
+      TM.getMCSubtargetInfo()->hasFeature(SystemZ::FeatureVector) &&
+      !TM.getMCSubtargetInfo()->hasFeature(SystemZ::FeatureSoftFloat);
+  MCSymbol *FuncExit = OutContext.getOrCreateSymbol(
+      HasVectorFeature ? "__xray_FunctionExitVec" : "__xray_FunctionExit");
+  MCSymbol *BeginOfSled = OutContext.createTempSymbol("xray_sled_", true);
+  OutStreamer->emitLabel(BeginOfSled);
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(SystemZ::BR).addReg(SystemZ::R14D));
+  EmitNop(OutContext, *OutStreamer, 4, getSubtargetInfo());
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(SystemZ::LLILF).addReg(SystemZ::R2D).addImm(0));
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(SystemZ::J)
+                     .addExpr(MCSymbolRefExpr::create(
+                         FuncExit, MCSymbolRefExpr::VK_PLT, OutContext)));
+  if (FallthroughLabel)
+    OutStreamer->emitLabel(FallthroughLabel);
+  recordSled(BeginOfSled, MI, SledKind::FUNCTION_EXIT, 2);
+}
+
 // The *alignment* of 128-bit vector types is different between the software
 // and hardware vector ABIs. If the there is an externally visible use of a
 // vector type in the module it should be annotated with an attribute.

llvm/lib/Target/SystemZ/SystemZAsmPrinter.h

@@ -111,6 +111,15 @@ class LLVM_LIBRARY_VISIBILITY SystemZAsmPrinter : public AsmPrinter {
   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
                              const char *ExtraCode, raw_ostream &OS) override;
 
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    AsmPrinter::runOnMachineFunction(MF);
+
+    // Emit the XRay table for this function.
+    emitXRayTable();
+
+    return false;
+  }
+
   bool doInitialization(Module &M) override {
     SM.reset();
     return AsmPrinter::doInitialization(M);
@@ -124,6 +133,9 @@ class LLVM_LIBRARY_VISIBILITY SystemZAsmPrinter : public AsmPrinter {
   void LowerFENTRY_CALL(const MachineInstr &MI, SystemZMCInstLower &MCIL);
   void LowerSTACKMAP(const MachineInstr &MI);
   void LowerPATCHPOINT(const MachineInstr &MI, SystemZMCInstLower &Lower);
+  void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
+                                     SystemZMCInstLower &Lower);
+  void LowerPATCHABLE_RET(const MachineInstr &MI, SystemZMCInstLower &Lower);
   void emitAttributes(Module &M);
 };
 } // end namespace llvm

llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp

@@ -30,6 +30,7 @@
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/CodeGen/VirtRegMap.h"
 #include "llvm/MC/MCInstrDesc.h"
@@ -1792,6 +1793,10 @@ unsigned SystemZInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
     return MI.getOperand(1).getImm();
   else if (MI.getOpcode() == SystemZ::FENTRY_CALL)
     return 6;
+  if (MI.getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER)
+    return 18;
+  if (MI.getOpcode() == TargetOpcode::PATCHABLE_RET)
+    return 18 + (MI.getOperand(0).getImm() == SystemZ::CondReturn ? 4 : 0);
 
   return MI.getDesc().getSize();
 }

llvm/lib/Target/SystemZ/SystemZSubtarget.h

@@ -106,6 +106,8 @@ class SystemZSubtarget : public SystemZGenSubtargetInfo {
   bool GETTER() const { return ATTRIBUTE; }
 #include "SystemZGenSubtargetInfo.inc"
 
+  bool isXRaySupported() const override { return true; }
+
   bool isAddressedViaADA(const GlobalValue *GV) const;
 
   // Return true if GV can be accessed using LARL for reloc model RM

llvm/test/CodeGen/SystemZ/xray.ll

@@ -0,0 +1,44 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck --check-prefixes=CHECK,NOVECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck --check-prefixes=CHECK,VECTOR %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 -mattr=+soft-float | FileCheck --check-prefixes=CHECK,NOVECTOR %s
+
+
+define signext i32 @foo() "function-instrument"="xray-always" {
+; CHECK-LABEL: .Lxray_sled_0:
+; CHECK:         j	.Ltmp[[#l:]]
+; CHECK:         bcr	0, %r0
+; CHECK:         llilf	%r2, 0
+; NOVECTOR:      brasl	%r14, __xray_FunctionEntry@PLT
+; VECTOR:        brasl	%r14, __xray_FunctionEntryVec@PLT
+; CHECK:       .Ltmp[[#l]]:
+  ret i32 0
+; CHECK-LABEL: .Lxray_sled_1:
+; CHECK:         br	%r14
+; CHECK:         bc	0, 0
+; CHECK:         llilf	%r2, 0
+; NOVECtOR:      j	__xray_FunctionExit@PLT
+; VECTOR:        j	__xray_FunctionExitVec@PLT
+}
+
+; CHECK: 	.section	xray_instr_map,"ao",@progbits,foo
+; CHECK: .Lxray_sleds_start0:
+; CHECK: [[TMP1:.Ltmp[0-9]+]]:
+; CHECK: 	.quad	.Lxray_sled_0-[[TMP1]]
+; CHECK: 	.quad	.Lfunc_begin0-([[TMP1]]+8)
+; CHECK: 	.byte	0x00
+; CHECK: 	.byte	0x01
+; CHECK: 	.byte	0x02
+; CHECK:  .space	13
+; CHECK: [[TMP2:.Ltmp[0-9]+]]:
+; CHECK: 	.quad	.Lxray_sled_1-[[TMP2]]
+; CHECK: 	.quad	.Lfunc_begin0-([[TMP2]]+8)
+; CHECK: 	.byte	0x01
+; CHECK: 	.byte	0x01
+; CHECK: 	.byte	0x02
+; CHECK: 	.space	13
+; CHECK: .Lxray_sleds_end0:
+; CHECK: 	.section	xray_fn_idx,"ao",@progbits,foo
+; CHECK: 	.p2align	4
+; CHECK: .Lxray_fn_idx0:
+; CHECK: 	.quad	.Lxray_sleds_start0-.Lxray_fn_idx0
+; CHECK: 	.quad	2