Local reducer tables

.github/workflows/llvm-project-tests.yml

@@ -28,7 +28,7 @@ concurrency:
 env:
   # Workaround for https://github.com/actions/virtual-environments/issues/5900.
   # This should be a no-op for non-mac OSes
-  CPLUS_INCLUDE_PATH: /usr/local/Cellar/llvm/15.0.7_1/include/c++/v1:/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include
+  CPLUS_INCLUDE_PATH: /usr/local/Cellar/llvm@15/15.0.7/include/c++/v1:/usr/local/Cellar/llvm/15.0.7_1/include/c++/v1:/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include
 
 jobs:
   lit-tests:

clang/include/clang/Basic/Builtins.def

@@ -1691,7 +1691,7 @@ LANGBUILTIN(__arithmetic_fence, "v.", "t", ALL_LANGUAGES)
 
 // Tapir.  Rewriting of reducer references happens during sema
 // and needs a builtin to carry the information to codegen.
-BUILTIN(__hyper_lookup, "v*vC*", "nU")
+BUILTIN(__hyper_lookup, "v*vC*z.", "nU")
 
 #undef BUILTIN
 #undef LIBBUILTIN

clang/lib/CodeGen/CGBuiltin.cpp

@@ -5438,8 +5438,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     return RValue::get(Ptr);
   }
   case Builtin::BI__hyper_lookup: {
-    Function *F = CGM.getIntrinsic(Intrinsic::hyper_lookup);
-    return RValue::get(Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))}));
+    llvm::Value *Size = EmitScalarExpr(E->getArg(1));
+    Function *F = CGM.getIntrinsic(Intrinsic::hyper_lookup, Size->getType());
+    llvm::Value *Ptr = EmitScalarExpr(E->getArg(0));
+    llvm::Value *Identity = EmitScalarExpr(E->getArg(2));
+    llvm::Value *Reduce = EmitScalarExpr(E->getArg(3));
+    return RValue::get(Builder.CreateCall(
+        F, {Ptr, Size, Builder.CreateBitCast(Identity, VoidPtrTy),
+            Builder.CreateBitCast(Reduce, VoidPtrTy)}));
   }
   }
   IsSpawnedScope SpawnedScp(this);

clang/lib/Sema/SemaExpr.cpp

@@ -2033,12 +2033,10 @@ Expr *Sema::BuildHyperobjectLookup(Expr *E, bool Pointer) {
 
   // For now all hyperobjects use the same lookup function.
   IdentifierInfo *ID = PP.getIdentifierInfo("__hyper_lookup");
-  ValueDecl *Builtin =
-      dyn_cast<ValueDecl>
-      (LazilyCreateBuiltin(ID, ID->getBuiltinID(),
-                           /* Scope = */ nullptr,
-                           /* ForRedeclaration = */ false,
-                           SourceLocation()));
+  ValueDecl *Builtin = dyn_cast<ValueDecl>(
+      LazilyCreateBuiltin(ID, ID->getBuiltinID(),
+                          /* Scope = */ nullptr,
+                          /* ForRedeclaration = */ false, SourceLocation()));
   // __hyper_lookup must be defined in Builtins.def.
   assert(Builtin && "no __hyper_lookup builtin");
 
@@ -2049,59 +2047,62 @@ Expr *Sema::BuildHyperobjectLookup(Expr *E, bool Pointer) {
       InputType.getLocalFastQualifiers());
   QualType Ptr = Context.getPointerType(ResultType);
 
+  ExprResult SizeExpr;
+  if (ResultType.getTypePtr()->isDependentType()) {
+    SizeExpr = CreateUnaryExprOrTypeTraitExpr(E, E->getExprLoc(), UETT_SizeOf);
+  } else {
+    QualType SizeType = Context.getSizeType();
+    llvm::APInt Size(Context.getTypeSize(SizeType),
+                     Context.getTypeSizeInChars(ResultType).getQuantity());
+    SizeExpr = IntegerLiteral::Create(Context, Size, SizeType, E->getExprLoc());
+  }
+
   Expr *VarAddr;
-  if (Pointer)
+  if (Pointer) {
     VarAddr = E;
-  else if (Difficult) {
+  } else if (Difficult) {
     IdentifierInfo *BAI = PP.getIdentifierInfo("__builtin_addressof");
-    ValueDecl *BAV =
-      dyn_cast<ValueDecl>
-      (LazilyCreateBuiltin(BAI, BAI->getBuiltinID(),
-                           /* Scope = */ nullptr,
-                           /* ForRedeclaration = */ false,
-                           SourceLocation()));
+    ValueDecl *BAV = dyn_cast<ValueDecl>(
+        LazilyCreateBuiltin(BAI, BAI->getBuiltinID(),
+                            /* Scope = */ nullptr,
+                            /* ForRedeclaration = */ false, SourceLocation()));
     assert(BAV && "no __builtin_addressof builtin");
-    DeclRefExpr *BAD = BuildDeclRefExpr(BAV, Builtin->getType(),
-                                        VK_PRValue, SourceLocation(),
-                                        nullptr);
+    DeclRefExpr *BAD = BuildDeclRefExpr(BAV, Builtin->getType(), VK_PRValue,
+                                        SourceLocation(), nullptr);
 
-    ExprResult Address = BuildCallExpr(nullptr, BAD, E->getExprLoc(),
-                                       { E }, E->getExprLoc(), nullptr);
+    ExprResult Address = BuildCallExpr(nullptr, BAD, E->getExprLoc(), {E},
+                                       E->getExprLoc(), nullptr);
     assert(Address.isUsable());
     VarAddr = Address.get();
   } else {
-    VarAddr = UnaryOperator::Create(Context, E, UO_AddrOf, Ptr,
-                                    VK_PRValue, OK_Ordinary,
-                                    SourceLocation(), false,
+    VarAddr = UnaryOperator::Create(Context, E, UO_AddrOf, Ptr, VK_PRValue,
+                                    OK_Ordinary, SourceLocation(), false,
                                     CurFPFeatureOverrides());
   }
-  ExprResult Call = BuildCallExpr(nullptr, Lookup, E->getExprLoc(),
-				  { VarAddr }, E->getExprLoc(), nullptr);
+  Expr *CallArgs[] = {VarAddr, SizeExpr.get(), HT->getIdentity(),
+                      HT->getReduce()};
+  ExprResult Call = BuildCallExpr(nullptr, Lookup, E->getExprLoc(), CallArgs,
+                                  E->getExprLoc(), nullptr);
 
-  // Template expansion normally strips out implicit casts,
-  // so make this explicit in C++.
+  // Template expansion normally strips out implicit casts, so make this
+  // explicit in C++.
   CastExpr *Casted = nullptr;
   if (Difficult)
-    Casted = CXXStaticCastExpr::Create(Context, Ptr, VK_PRValue,
-                                       CK_BitCast, Call.get(), nullptr,
-                                       Context.CreateTypeSourceInfo(Ptr),
-                                       FPOptionsOverride(),
-                                       SourceLocation(),
-                                       SourceLocation(),
-                                       SourceRange());
+    Casted = CXXStaticCastExpr::Create(
+        Context, Ptr, VK_PRValue, CK_BitCast, Call.get(), nullptr,
+        Context.CreateTypeSourceInfo(Ptr), FPOptionsOverride(),
+        SourceLocation(), SourceLocation(), SourceRange());
   else
-    Casted = ImplicitCastExpr::Create(Context, Ptr,
-                                      CK_BitCast,
-                                      Call.get(), nullptr, VK_PRValue,
-                                      CurFPFeatureOverrides());
+    Casted =
+        ImplicitCastExpr::Create(Context, Ptr, CK_BitCast, Call.get(), nullptr,
+                                 VK_PRValue, CurFPFeatureOverrides());
 
   if (Pointer)
     return Casted;
 
-  auto *Deref =
-    UnaryOperator::Create(Context, Casted, UO_Deref, ResultType,
-                          VK_LValue, OK_Ordinary, SourceLocation(),
-                          false, CurFPFeatureOverrides());
+  auto *Deref = UnaryOperator::Create(Context, Casted, UO_Deref, ResultType,
+                                      VK_LValue, OK_Ordinary, SourceLocation(),
+                                      false, CurFPFeatureOverrides());
 
   return Deref;
 }

clang/test/Cilk/hyper-array-extern-1.cpp

@@ -11,7 +11,7 @@ int read_array_hyper(unsigned i)
   return x[i];
   // CHECK: %[[ARRAYIDX:.+]] = getelementptr inbounds
   // CHECK: %[[KEY:.+]] = bitcast i32* %[[ARRAYIDX]] to i8*
-  // CHECK: %[[VIEWRAW:.+]] = call i8* @llvm.hyper.lookup(i8* %[[KEY]])
+  // CHECK: %[[VIEWRAW:.+]] = call i8* @llvm.hyper.lookup.i64(i8* %[[KEY]], i64 4, i8* null, i8* null)
   // CHECK-NOT: call i8* @llvm.hyper.lookup
   // CHECK: %[[VIEW:.+]] = bitcast i8* %[[VIEWRAW]] to i32*
   // CHECK: %[[VAL:.+]] = load i32, i32* %[[VIEW]]

clang/test/Cilk/hyper-assign.c

@@ -6,32 +6,32 @@ extern long _Hyperobject x, _Hyperobject y;
 
 long chain_assign()
 {
-  // CHECK: %[[Y1RAW:.+]] = call i8* @llvm.hyper.lookup(i8* bitcast (i64* @y to i8*))
+  // CHECK: %[[Y1RAW:.+]] = call i8* @llvm.hyper.lookup.i64(i8* bitcast (i64* @y to i8*), i64 8, i8* null, i8* null)
   // CHECK: %[[Y1PTR:.+]] = bitcast i8* %[[Y1RAW]] to i64*
   // CHECK: %[[Y1VAL:.+]] = load i64, i64* %[[Y1PTR]]
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i64* @x to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i64* @x to i8*), i64 8, i8* null, i8* null)
   // CHECK: store i64 %[[Y1VAL]]
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i64* @y to i8*))
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i64* @x to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i64* @y to i8*), i64 8, i8* null, i8* null)
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i64* @x to i8*), i64 8, i8* null, i8* null)
   return x = y = x = y;
 }
 
 long simple_assign(long val)
 {
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i64* @x to i8*))
-  // CHECK-NOT: call i8* @llvm.hyper.lookup(i8* bitcast (i64* @x to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i64* @x to i8*), i64 8, i8* null, i8* null)
+  // CHECK-NOT: call i8* @llvm.hyper.lookup
   // CHECK: store i64
   return x = val;
 }
 
 long subtract()
 {
   // The order is not fixed here.
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i64* @y to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i64* @y to i8*), i64 8, i8* null, i8* null)
   // CHECK: load i64
   // CHECK: add nsw i64 %[[Y:.+]], 1
   // CHECK: store i64
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i64* @x to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i64* @x to i8*), i64 8, i8* null, i8* null)
   // CHECK: load i64
   // CHECK: sub nsw
   // CHECK: store i64

clang/test/Cilk/hyper-complex.c

@@ -7,7 +7,7 @@ extern __complex__ float _Hyperobject c;
 // CHECK-LABEL: get_real 
 float get_real()
 {
-  // CHECK: %[[RAW1:.+]] = call i8* @llvm.hyper.lookup(i8* bitcast ({ float, float }* @c to i8*))
+  // CHECK: %[[RAW1:.+]] = call i8* @llvm.hyper.lookup.i64(i8* bitcast ({ float, float }* @c to i8*), i64 8, i8* null, i8* null)
   // CHECK: %[[VIEW1:.+]] = bitcast i8* %[[RAW1]] to { float, float }*
   // CHECK: %[[FIELD1:.+]] = getelementptr inbounds { float, float }, { float, float }* %[[VIEW1]], i32 0, i32 0
   // CHECK: %[[RET1:.+]] = load float, float* %[[FIELD1]]
@@ -17,7 +17,7 @@ float get_real()
 // CHECK-LABEL: get_imag
 float get_imag()
 {
-  // CHECK: %[[RAW2:.+]] = call i8* @llvm.hyper.lookup(i8* bitcast ({ float, float }* @c to i8*))
+  // CHECK: %[[RAW2:.+]] = call i8* @llvm.hyper.lookup.i64(i8* bitcast ({ float, float }* @c to i8*), i64 8, i8* null, i8* null)
   // CHECK: %[[VIEW2:.+]] = bitcast i8* %[[RAW2]] to { float, float }*
   // CHECK: %[[FIELD2:.+]] = getelementptr inbounds { float, float }, { float, float }* %[[VIEW2]], i32 0, i32 1
   // CHECK: load float, float* %[[FIELD2]]
@@ -29,7 +29,7 @@ float get_imag()
 float get_abs()
 {
   // Only one call to llvm.hyper.lookup.
-  // CHECK: @llvm.hyper.lookup(i8* bitcast ({ float, float }* @c to i8*))
+  // CHECK: @llvm.hyper.lookup.i64(i8* bitcast ({ float, float }* @c to i8*), i64 8, i8* null, i8* null)
   // CHECK-NOT: @llvm.hyper.lookup
   // CHECK: call float @cabsf
   // CHECK: ret float

clang/test/Cilk/hyper-copy.c

@@ -9,9 +9,9 @@ extern struct S b __attribute__((aligned(8)));
 // CHECK-LABEL: scopy
 void scopy()
 {
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (%struct.S* @a to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (%struct.S* @a to i8*), i64 8, i8* null, i8* null)
   // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 bitcast (%struct.S* @b to i8*)
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (%struct.S* @a to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (%struct.S* @a to i8*), i64 8, i8* null, i8* null)
   // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64
   // CHECK: ret void
   b = a;

clang/test/Cilk/hyper-template.cpp

@@ -5,7 +5,7 @@ template<typename T> struct S { T member; };
 S<long> _Hyperobject S_long;
 
 // CHECK-LABEL: @_Z1fv
-// CHECK: %0 = call i8* @llvm.hyper.lookup(i8* bitcast (%struct.S* @S_long to i8*))
+// CHECK: %0 = call i8* @llvm.hyper.lookup.i64(i8* bitcast (%struct.S* @S_long to i8*), i64 8, i8* null, i8* null)
 // CHECK-NOT: call i8* @llvm.hyper.lookup
 // CHECK: getelementptr
 // CHECK: %[[RET:.+]] = load i64

clang/test/Cilk/hyper-unary.c

@@ -11,10 +11,10 @@ void function1()
 {
   // CHECK: store i32 1, i32* %[[Y:.+]],
   int _Hyperobject y = 1;
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i32* @x to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i32* @x to i8*), i64 4, i8* null, i8* null)
   // CHECK: load i32
   // CHECK: %[[Y1:.+]] = bitcast i32* %[[Y]] to i8*
-  // CHECK: call i8* @llvm.hyper.lookup(i8* %[[Y1]])
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* %[[Y1]], i64 4, i8* null, i8* null)
   // CHECK: load i32
   (void)x; (void)y;
 }
@@ -24,10 +24,10 @@ void function2()
 {
   // CHECK: store i32 1, i32* %[[Y:.+]],
   int _Hyperobject y = 1;
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i32* @x to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i32* @x to i8*), i64 4, i8* null, i8* null)
   // CHECK: load i32
   // CHECK: %[[Y2:.+]] = bitcast i32* %[[Y]] to i8*
-  // CHECK: call i8* @llvm.hyper.lookup(i8* %[[Y2]])
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* %[[Y2]], i64 4, i8* null, i8* null)
   // CHECK: load i32
   (void)!x; (void)!y;
 }
@@ -37,21 +37,21 @@ void function3()
 {
   // CHECK: store i32 1, i32* %[[Y:.+]],
   int _Hyperobject y = 1;
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i32* @x to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i32* @x to i8*), i64 4, i8* null, i8* null)
   // CHECK: load i32
   // CHECK: %[[Y3:.+]] = bitcast i32* %[[Y]] to i8*
-  // CHECK: call i8* @llvm.hyper.lookup(i8* %[[Y3]])
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* %[[Y3]], i64 4, i8* null, i8* null)
   // CHECK: load i32
   (void)-x; (void)-y;
-  // CHECK: call i8* @llvm.hyper.lookup(i8* bitcast (i32* @x to i8*))
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* bitcast (i32* @x to i8*), i64 4, i8* null, i8* null)
   // CHECK: load i32
   // CHECK: %[[Y4:.+]] = bitcast i32* %[[Y]] to i8*
-  // CHECK: call i8* @llvm.hyper.lookup(i8* %[[Y4]])
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* %[[Y4]], i64 4, i8* null, i8* null)
   // CHECK: load i32
   (void)~x; (void)~y;
   // CHECK: %[[XP:.+]] = load i32*, i32** @xp
   // CHECK: %[[XP1:.+]] = bitcast i32* %[[XP]] to i8*
-  // CHECK: call i8* @llvm.hyper.lookup(i8* %[[XP1]])
+  // CHECK: call i8* @llvm.hyper.lookup.i64(i8* %[[XP1]], i64 4, i8* null, i8* null)
   // CHECK: load i32
   (void)*xp;
 }

llvm/include/llvm/IR/Intrinsics.td

@@ -1386,9 +1386,11 @@ def int_task_frameaddress
 // Ideally the types would be [llvm_anyptr_ty], [LLVMMatchType<0>]
 // but that does not work, so rely on the front end to insert bitcasts.
 def int_hyper_lookup
-    : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty],
-    [IntrWillReturn, IntrReadMem, IntrInaccessibleMemOnly,
-     IntrStrandPure, IntrHyperView, IntrInjective]>;
+    : Intrinsic<[llvm_ptr_ty],
+                [llvm_ptr_ty, llvm_anyint_ty, llvm_ptr_ty, llvm_ptr_ty], [
+                  IntrWillReturn, IntrReadMem, IntrInaccessibleMemOnly,
+                  IntrStrandPure, IntrHyperView, IntrInjective
+                ]>;
 
 // TODO: Change tablegen to allow function pointer types in intrinsics.
 def int_reducer_register

llvm/lib/Analysis/BasicAliasAnalysis.cpp

@@ -1662,7 +1662,7 @@ static const Value *getRecognizedArgument(const Value *V, bool InSameSpindle,
   if (!C)
     return nullptr;
   unsigned NumOperands = C->getNumOperands();
-  if (NumOperands != 2)
+  if (NumOperands != 2 && NumOperands != 5)
     return nullptr;
   for (auto E : TapirFnAttrTable) {
     if (C->hasFnAttr(E.first))

llvm/lib/Transforms/Instrumentation/CilkSanitizer.cpp

@@ -3867,6 +3867,24 @@ bool CilkSanitizerImpl::instrumentIntrinsicCall(
     AfterHookParamTys.push_back(ArgSpill->getType());
     AfterHookParamVals.push_back(ArgSpill);
   }
+
+  // Special-case intrinsics.
+  IntrinsicInst *II = dyn_cast<IntrinsicInst>(I);
+  switch (II->getIntrinsicID()) {
+  case Intrinsic::hyper_lookup: {
+    FunctionType *AfterHookTy =
+        FunctionType::get(IRB.getInt8PtrTy(), AfterHookParamTys, Called->isVarArg());
+    FunctionCallee AfterIntrinCallHook =
+        getOrInsertSynthesizedHook(("__csan_" + Buf).str(), AfterHookTy, FnAttrs);
+    CallInst *HookCall =
+        insertHookCall(&*Iter, AfterIntrinCallHook, AfterHookParamVals);
+    II->replaceUsesWithIf(HookCall, [HookCall](Use &U) {
+      return cast<Instruction>(U.getUser()) != HookCall;
+    });
+    return true;
+  }
+  }
+
   FunctionType *AfterHookTy =
       FunctionType::get(IRB.getVoidTy(), AfterHookParamTys, Called->isVarArg());
   FunctionCallee AfterIntrinCallHook =

llvm/lib/Transforms/Tapir/OpenCilkABI.cpp

@@ -223,7 +223,8 @@ void OpenCilkABI::prepareModule() {
   FunctionType *Grainsize16FnTy = FunctionType::get(Int16Ty, {Int16Ty}, false);
   FunctionType *Grainsize32FnTy = FunctionType::get(Int32Ty, {Int32Ty}, false);
   FunctionType *Grainsize64FnTy = FunctionType::get(Int64Ty, {Int64Ty}, false);
-  FunctionType *PtrPtrTy = FunctionType::get(VoidPtrTy, {VoidPtrTy}, false);
+  FunctionType *LookupTy = FunctionType::get(
+      VoidPtrTy, {VoidPtrTy, Int64Ty, VoidPtrTy, VoidPtrTy}, false);
   FunctionType *UnregTy = FunctionType::get(VoidTy, {VoidPtrTy}, false);
   FunctionType *Reg32Ty =
       FunctionType::get(VoidTy, {VoidPtrTy, Int32Ty, VoidPtrTy,
@@ -258,7 +259,7 @@ void OpenCilkABI::prepareModule() {
        CilkRTSCilkForGrainsize32},
       {"__cilkrts_cilk_for_grainsize_64", Grainsize64FnTy,
        CilkRTSCilkForGrainsize64},
-      {"__cilkrts_reducer_lookup", PtrPtrTy, CilkRTSReducerLookup},
+      {"__cilkrts_reducer_lookup", LookupTy, CilkRTSReducerLookup},
       {"__cilkrts_reducer_register_32", Reg32Ty, CilkRTSReducerRegister32},
       {"__cilkrts_reducer_register_64", Reg64Ty, CilkRTSReducerRegister64},
       {"__cilkrts_reducer_unregister", UnregTy, CilkRTSReducerUnregister},