Skip to content

Commit

Permalink
Preserve metadata in scalarization pass
Browse files Browse the repository at this point in the history
Preserve metadata in scalarization pass
  • Loading branch information
krzysiek358 authored and igcbot committed Dec 3, 2024
1 parent 11c4dfd commit 6b857dc
Show file tree
Hide file tree
Showing 15 changed files with 337 additions and 24 deletions.
52 changes: 30 additions & 22 deletions IGC/Compiler/Optimizer/Scalarizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -410,17 +410,16 @@ void ScalarizeFunction::visitUnaryOperator(UnaryOperator& UI)
newScalarizedInsts.resize(numElements);
for (unsigned dup = 0; dup < numElements; dup++)
{
Value* Val = UnaryOperator::Create(
UnaryOperator* Val = UnaryOperator::Create(
UI.getOpcode(),
operand0[dup],
UI.getName(),
&UI
);
if (UnaryOperator* UO = dyn_cast<UnaryOperator>(Val)) {
// Copy fast math flags if any.
if (isa<FPMathOperator>(UO))
UO->setFastMathFlags(UI.getFastMathFlags());
}
// Copy fast math flags if any.
if (isa<FPMathOperator>(Val))
Val->setFastMathFlags(UI.getFastMathFlags());
Val->copyMetadata(UI);
newScalarizedInsts[dup] = Val;
}

Expand Down Expand Up @@ -461,26 +460,27 @@ void ScalarizeFunction::visitBinaryOperator(BinaryOperator& BI)
newScalarizedInsts.resize(numElements);
for (unsigned dup = 0; dup < numElements; dup++)
{
Value* Val = BinaryOperator::Create(
BinaryOperator* Val = BinaryOperator::Create(
BI.getOpcode(),
operand0[dup],
operand1[dup],
BI.getName(),
&BI
);
if (BinaryOperator* BO = dyn_cast<BinaryOperator>(Val)) {
// Copy overflow flags if any.
if (isa<OverflowingBinaryOperator>(BO)) {
BO->setHasNoSignedWrap(BI.hasNoSignedWrap());
BO->setHasNoUnsignedWrap(BI.hasNoUnsignedWrap());
}
// Copy exact flag if any.
if (isa<PossiblyExactOperator>(BO))
BO->setIsExact(BI.isExact());
// Copy fast math flags if any.
if (isa<FPMathOperator>(BO))
BO->setFastMathFlags(BI.getFastMathFlags());

// Copy overflow flags if any.
if (isa<OverflowingBinaryOperator>(Val)) {
Val->setHasNoSignedWrap(BI.hasNoSignedWrap());
Val->setHasNoUnsignedWrap(BI.hasNoUnsignedWrap());
}
// Copy exact flag if any.
if (isa<PossiblyExactOperator>(Val))
Val->setIsExact(BI.isExact());
// Copy fast math flags if any.
if (isa<FPMathOperator>(Val))
Val->setFastMathFlags(BI.getFastMathFlags());

Val->copyMetadata(BI);
newScalarizedInsts[dup] = Val;
}

Expand Down Expand Up @@ -534,6 +534,7 @@ void ScalarizeFunction::visitCmpInst(CmpInst& CI)
{
Val->setFastMathFlags(CI.getFastMathFlags());
}
Val->copyMetadata(CI);
newScalarizedInsts[dup] = Val;
}

Expand Down Expand Up @@ -599,13 +600,15 @@ void ScalarizeFunction::visitCastInst(CastInst& CI)
newScalarizedInsts.resize(numElements);
for (unsigned dup = 0; dup < numElements; dup++)
{
newScalarizedInsts[dup] = CastInst::Create(
CastInst* tmp = CastInst::Create(
CI.getOpcode(),
operand0[dup],
scalarDestType,
CI.getName(),
&CI
);
tmp->copyMetadata(CI);
newScalarizedInsts[dup] = tmp;
}

// Add new value/s to SCM
Expand Down Expand Up @@ -715,6 +718,8 @@ void ScalarizeFunction::visitPHINode(PHINode& PI)
tmp->setFastMathFlags(PI.getFastMathFlags());
}

tmp->copyMetadata(PI);

newScalarizedPHI[i] = tmp;
}

Expand Down Expand Up @@ -800,6 +805,7 @@ void ScalarizeFunction::visitSelectInst(SelectInst& SI)
{
Val->setFastMathFlags(SI.getFastMathFlags());
}
Val->copyMetadata(SI);
newScalarizedInsts[dup] = Val;
}
else
Expand Down Expand Up @@ -1007,12 +1013,13 @@ void ScalarizeFunction::visitCallInst(CallInst& CI)
for (unsigned i = 0; i < numElements; i++)
{
Type* scalarType = operand0[i]->getType();
Value* scalarFshl = CallInst::Create(
CallInst* scalarFshl = CallInst::Create(
Intrinsic::getDeclaration(CI.getModule(), Intrinsic::fshl, { scalarType }),
{ operand0[i], operand1[i], operand2[i] },
CI.getName() + ".scalar",
&CI
);
scalarFshl->copyMetadata(CI);
newScalarizedInsts[i] = scalarFshl;
}

Expand Down Expand Up @@ -1077,8 +1084,9 @@ void ScalarizeFunction::visitGetElementPtrInst(GetElementPtrInst& GI)
auto op2 = indexValue->getType()->isVectorTy() ? operand2[i] : indexValue;

Type* BaseTy = GI.getSourceElementType();
Value* newGEP = GetElementPtrInst::Create(BaseTy, op1, op2,
GetElementPtrInst* newGEP = GetElementPtrInst::Create(BaseTy, op1, op2,
VALUE_NAME(GI.getName()), &GI);
newGEP->copyMetadata(GI);
Value* constIndex = ConstantInt::get(Type::getInt32Ty(context()), i);
Instruction* insert = InsertElementInst::Create(assembledVector,
newGEP, constIndex,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,27 @@ define spir_kernel void @basic(<2 x i32> %src1, <2 x i32> %src2) {
ret void
}

define spir_kernel void @should_preserve_metadata(<2 x i32> %src1, <2 x i32> %src2) {
; CHECK-LABEL: define spir_kernel void @should_preserve_metadata(
; CHECK-SAME: <2 x i32> [[SRC1:%.*]], <2 x i32> [[SRC2:%.*]]) {
; CHECK-NEXT: [[SRC2_SCALAR:%.*]] = extractelement <2 x i32> [[SRC2]], i32 0
; CHECK-NEXT: [[SRC2_SCALAR2:%.*]] = extractelement <2 x i32> [[SRC2]], i32 1
; CHECK-NEXT: [[SRC1_SCALAR:%.*]] = extractelement <2 x i32> [[SRC1]], i32 0
; CHECK-NEXT: [[SRC1_SCALAR1:%.*]] = extractelement <2 x i32> [[SRC1]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = alloca <2 x i32>, align 8
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[SRC1_SCALAR]], [[SRC2_SCALAR]], !any_metadata [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[SRC1_SCALAR1]], [[SRC2_SCALAR2]], !any_metadata [[META0]]
; CHECK-NEXT: [[DOTASSEMBLED_VECT:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i32 0
; CHECK-NEXT: [[DOTASSEMBLED_VECT3:%.*]] = insertelement <2 x i32> [[DOTASSEMBLED_VECT]], i32 [[TMP3]], i32 1
; CHECK-NEXT: store <2 x i32> [[DOTASSEMBLED_VECT3]], <2 x i32>* [[TMP1]], align 8
; CHECK-NEXT: ret void
;
%1 = alloca <2 x i32>
%2 = add <2 x i32> %src1, %src2, !any_metadata !{i32 0}
store <2 x i32> %2, <2 x i32>* %1
ret void
}

define spir_kernel void @should_work_with_different_instruction_type(<2 x float> %src1, <2 x float> %src2) {
; CHECK-LABEL: define spir_kernel void @should_work_with_different_instruction_type(
; CHECK-SAME: <2 x float> [[SRC1:%.*]], <2 x float> [[SRC2:%.*]]) {
Expand Down Expand Up @@ -255,3 +276,5 @@ define spir_kernel void @should_work_with_nuw_nsw(<2 x i32> %src1, <2 x i32> %sr
store <2 x i32> %2, <2 x i32>* %1
ret void
}

; CHECK: [[META0]] = !{i32 0}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,27 @@ define spir_kernel void @basic(<2 x i32> %src1, <2 x i32> %src2) {
ret void
}

define spir_kernel void @should_preserve_metadata(<2 x i32> %src1, <2 x i32> %src2) {
; CHECK-LABEL: define spir_kernel void @should_preserve_metadata(
; CHECK-SAME: <2 x i32> [[SRC1:%.*]], <2 x i32> [[SRC2:%.*]]) {
; CHECK-NEXT: [[SRC2_SCALAR:%.*]] = extractelement <2 x i32> [[SRC2]], i32 0
; CHECK-NEXT: [[SRC2_SCALAR2:%.*]] = extractelement <2 x i32> [[SRC2]], i32 1
; CHECK-NEXT: [[SRC1_SCALAR:%.*]] = extractelement <2 x i32> [[SRC1]], i32 0
; CHECK-NEXT: [[SRC1_SCALAR1:%.*]] = extractelement <2 x i32> [[SRC1]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = alloca <2 x i32>, align 8
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[SRC1_SCALAR]], [[SRC2_SCALAR]], !any_metadata [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[SRC1_SCALAR1]], [[SRC2_SCALAR2]], !any_metadata [[META0]]
; CHECK-NEXT: [[DOTASSEMBLED_VECT:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i32 0
; CHECK-NEXT: [[DOTASSEMBLED_VECT3:%.*]] = insertelement <2 x i32> [[DOTASSEMBLED_VECT]], i32 [[TMP3]], i32 1
; CHECK-NEXT: store <2 x i32> [[DOTASSEMBLED_VECT3]], ptr [[TMP1]], align 8
; CHECK-NEXT: ret void
;
%1 = alloca <2 x i32>
%2 = add <2 x i32> %src1, %src2, !any_metadata !{i32 0}
store <2 x i32> %2, ptr %1
ret void
}

define spir_kernel void @should_work_with_different_instruction_type(<2 x float> %src1, <2 x float> %src2) {
; CHECK-LABEL: define spir_kernel void @should_work_with_different_instruction_type(
; CHECK-SAME: <2 x float> [[SRC1:%.*]], <2 x float> [[SRC2:%.*]]) {
Expand Down Expand Up @@ -257,3 +278,5 @@ define spir_kernel void @should_work_with_nuw_nsw(<2 x i32> %src1, <2 x i32> %sr
store <2 x i32> %2, ptr %1
ret void
}

; CHECK: [[META0]] = !{i32 0}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,21 @@ define <2 x i8> @basic(<2 x i32> %src1) {
ret <2 x i8> %1
}

define <2 x i8> @should_preserve_metadata(<2 x i32> %src1) {
; CHECK-LABEL: define <2 x i8> @should_preserve_metadata(
; CHECK-SAME: <2 x i32> [[SRC1:%.*]]) {
; CHECK-NEXT: [[SRC1_SCALAR:%.*]] = extractelement <2 x i32> [[SRC1]], i32 0
; CHECK-NEXT: [[SRC1_SCALAR1:%.*]] = extractelement <2 x i32> [[SRC1]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[SRC1_SCALAR]] to i8, !any_metadata [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[SRC1_SCALAR1]] to i8, !any_metadata [[META0]]
; CHECK-NEXT: [[DOTASSEMBLED_VECT:%.*]] = insertelement <2 x i8> undef, i8 [[TMP1]], i32 0
; CHECK-NEXT: [[DOTASSEMBLED_VECT2:%.*]] = insertelement <2 x i8> [[DOTASSEMBLED_VECT]], i8 [[TMP2]], i32 1
; CHECK-NEXT: ret <2 x i8> [[DOTASSEMBLED_VECT2]]
;
%1 = trunc <2 x i32> %src1 to <2 x i8>, !any_metadata !{i32 0}
ret <2 x i8> %1
}

define <2 x float> @should_work_with_different_instruction_type(<2 x double> %src1) {
; CHECK-LABEL: define <2 x float> @should_work_with_different_instruction_type(
; CHECK-SAME: <2 x double> [[SRC1:%.*]]) {
Expand Down Expand Up @@ -193,3 +208,5 @@ define i8 @should_not_scalarize_scalar() {
%1 = trunc i32 4 to i8
ret i8 %1
}

; CHECK: [[META0]] = !{i32 0}
17 changes: 17 additions & 0 deletions IGC/Compiler/tests/ScalarizeFunction/scalarize-cast-instruction.ll
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,21 @@ define <2 x i8> @basic(<2 x i32> %src1) {
ret <2 x i8> %1
}

define <2 x i8> @should_preserve_metadata(<2 x i32> %src1) {
; CHECK-LABEL: define <2 x i8> @should_preserve_metadata(
; CHECK-SAME: <2 x i32> [[SRC1:%.*]]) {
; CHECK-NEXT: [[SRC1_SCALAR:%.*]] = extractelement <2 x i32> [[SRC1]], i32 0
; CHECK-NEXT: [[SRC1_SCALAR1:%.*]] = extractelement <2 x i32> [[SRC1]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[SRC1_SCALAR]] to i8, !any_metadata [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[SRC1_SCALAR1]] to i8, !any_metadata [[META0]]
; CHECK-NEXT: [[DOTASSEMBLED_VECT:%.*]] = insertelement <2 x i8> undef, i8 [[TMP1]], i32 0
; CHECK-NEXT: [[DOTASSEMBLED_VECT2:%.*]] = insertelement <2 x i8> [[DOTASSEMBLED_VECT]], i8 [[TMP2]], i32 1
; CHECK-NEXT: ret <2 x i8> [[DOTASSEMBLED_VECT2]]
;
%1 = trunc <2 x i32> %src1 to <2 x i8>, !any_metadata !{i32 0}
ret <2 x i8> %1
}

define <2 x float> @should_work_with_different_instruction_type(<2 x double> %src1) {
; CHECK-LABEL: define <2 x float> @should_work_with_different_instruction_type(
; CHECK-SAME: <2 x double> [[SRC1:%.*]]) {
Expand Down Expand Up @@ -194,3 +209,5 @@ define i8 @should_not_scalarize_scalar() {
%1 = trunc i32 4 to i8
ret i8 %1
}

; CHECK: [[META0]] = !{i32 0}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,23 @@ define <2 x i1> @basic(<2 x i32> %src1, <2 x i32> %src2) {
ret <2 x i1> %1
}

define <2 x i1> @should_preserve_metadata(<2 x i32> %src1, <2 x i32> %src2) {
; CHECK-LABEL: define <2 x i1> @should_preserve_metadata(
; CHECK-SAME: <2 x i32> [[SRC1:%.*]], <2 x i32> [[SRC2:%.*]]) {
; CHECK-NEXT: [[SRC2_SCALAR:%.*]] = extractelement <2 x i32> [[SRC2]], i32 0
; CHECK-NEXT: [[SRC2_SCALAR2:%.*]] = extractelement <2 x i32> [[SRC2]], i32 1
; CHECK-NEXT: [[SRC1_SCALAR:%.*]] = extractelement <2 x i32> [[SRC1]], i32 0
; CHECK-NEXT: [[SRC1_SCALAR1:%.*]] = extractelement <2 x i32> [[SRC1]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[SRC1_SCALAR]], [[SRC2_SCALAR]], !any_metadata [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[SRC1_SCALAR1]], [[SRC2_SCALAR2]], !any_metadata [[META0]]
; CHECK-NEXT: [[DOTASSEMBLED_VECT:%.*]] = insertelement <2 x i1> undef, i1 [[TMP1]], i32 0
; CHECK-NEXT: [[DOTASSEMBLED_VECT3:%.*]] = insertelement <2 x i1> [[DOTASSEMBLED_VECT]], i1 [[TMP2]], i32 1
; CHECK-NEXT: ret <2 x i1> [[DOTASSEMBLED_VECT3]]
;
%1 = icmp eq <2 x i32> %src1, %src2, !any_metadata !{i32 0}
ret <2 x i1> %1
}

define <2 x i1> @should_work_with_different_instruction_type(<2 x float> %src1, <2 x float> %src2) {
; CHECK-LABEL: define <2 x i1> @should_work_with_different_instruction_type(
; CHECK-SAME: <2 x float> [[SRC1:%.*]], <2 x float> [[SRC2:%.*]]) {
Expand Down Expand Up @@ -212,3 +229,5 @@ define <2 x i1> @should_not_scalarize_two_constants() {
%1 = icmp eq <2 x i32> <i32 4, i32 4>, <i32 4, i32 8>
ret <2 x i1> %1
}

; CHECK: [[META0]] = !{i32 0}
19 changes: 19 additions & 0 deletions IGC/Compiler/tests/ScalarizeFunction/scalarize-comp-instruction.ll
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,23 @@ define <2 x i1> @basic(<2 x i32> %src1, <2 x i32> %src2) {
ret <2 x i1> %1
}

define <2 x i1> @should_preserve_metadata(<2 x i32> %src1, <2 x i32> %src2) {
; CHECK-LABEL: define <2 x i1> @should_preserve_metadata(
; CHECK-SAME: <2 x i32> [[SRC1:%.*]], <2 x i32> [[SRC2:%.*]]) {
; CHECK-NEXT: [[SRC2_SCALAR:%.*]] = extractelement <2 x i32> [[SRC2]], i32 0
; CHECK-NEXT: [[SRC2_SCALAR2:%.*]] = extractelement <2 x i32> [[SRC2]], i32 1
; CHECK-NEXT: [[SRC1_SCALAR:%.*]] = extractelement <2 x i32> [[SRC1]], i32 0
; CHECK-NEXT: [[SRC1_SCALAR1:%.*]] = extractelement <2 x i32> [[SRC1]], i32 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[SRC1_SCALAR]], [[SRC2_SCALAR]], !any_metadata [[META0:![0-9]+]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[SRC1_SCALAR1]], [[SRC2_SCALAR2]], !any_metadata [[META0]]
; CHECK-NEXT: [[DOTASSEMBLED_VECT:%.*]] = insertelement <2 x i1> undef, i1 [[TMP1]], i32 0
; CHECK-NEXT: [[DOTASSEMBLED_VECT3:%.*]] = insertelement <2 x i1> [[DOTASSEMBLED_VECT]], i1 [[TMP2]], i32 1
; CHECK-NEXT: ret <2 x i1> [[DOTASSEMBLED_VECT3]]
;
%1 = icmp eq <2 x i32> %src1, %src2, !any_metadata !{i32 0}
ret <2 x i1> %1
}

define <2 x i1> @should_work_with_different_instruction_type(<2 x float> %src1, <2 x float> %src2) {
; CHECK-LABEL: define <2 x i1> @should_work_with_different_instruction_type(
; CHECK-SAME: <2 x float> [[SRC1:%.*]], <2 x float> [[SRC2:%.*]]) {
Expand Down Expand Up @@ -214,3 +231,5 @@ define <2 x i1> @should_not_scalarize_two_constants() {
%1 = icmp eq <2 x i32> <i32 4, i32 4>, <i32 4, i32 8>
ret <2 x i1> %1
}

; CHECK: [[META0]] = !{i32 0}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,30 @@ define double @basic(<2 x double*> %pointers) {
ret double %return
}

define double @should_preserve_metadata(<2 x double*> %pointers) {
; CHECK-LABEL: define double @should_preserve_metadata(
; CHECK-SAME: <2 x double*> [[POINTERS:%.*]]) {
; CHECK-NEXT: [[POINTERS_SCALAR:%.*]] = extractelement <2 x double*> [[POINTERS]], i32 0
; CHECK-NEXT: [[POINTERS_SCALAR1:%.*]] = extractelement <2 x double*> [[POINTERS]], i32 1
; CHECK-NEXT: [[POINTER_TO_DOUBLE2:%.*]] = getelementptr double, double* [[POINTERS_SCALAR]], i32 1, !any_metadata [[META0:![0-9]+]]
; CHECK-NEXT: [[POINTER_TO_DOUBLE_ASSEMBLED_VECT:%.*]] = insertelement <2 x double*> undef, double* [[POINTER_TO_DOUBLE2]], i32 0
; CHECK-NEXT: [[POINTER_TO_DOUBLE3:%.*]] = getelementptr double, double* [[POINTERS_SCALAR1]], i32 1, !any_metadata [[META0]]
; CHECK-NEXT: [[POINTER_TO_DOUBLE_ASSEMBLED_VECT4:%.*]] = insertelement <2 x double*> [[POINTER_TO_DOUBLE_ASSEMBLED_VECT]], double* [[POINTER_TO_DOUBLE3]], i32 1
; CHECK-NEXT: [[VAL0:%.*]] = load double, double* [[POINTER_TO_DOUBLE2]], align 8
; CHECK-NEXT: [[VAL1:%.*]] = load double, double* [[POINTER_TO_DOUBLE3]], align 8
; CHECK-NEXT: [[RETURN:%.*]] = fadd double [[VAL0]], [[VAL1]]
; CHECK-NEXT: ret double [[RETURN]]
;
%pointer_to_double = getelementptr double, <2 x double*> %pointers, i32 1, !any_metadata !{i32 0}

%ptr0 = extractelement <2 x double*> %pointer_to_double, i32 0
%ptr1 = extractelement <2 x double*> %pointer_to_double, i32 1
%val0 = load double, double* %ptr0
%val1 = load double, double* %ptr1
%return = fadd double %val0, %val1
ret double %return
}

define double @should_work_with_vector_of_indices(<2 x double*> %pointers) {
; CHECK-LABEL: define double @should_work_with_vector_of_indices(
; CHECK-SAME: <2 x double*> [[POINTERS:%.*]]) {
Expand Down Expand Up @@ -187,4 +211,6 @@ define i64 @should_scalarize_only_vectors(%some_type* %pointer) {
%val1 = extractvalue %some_type %val, 0
%return = add i64 %val0, %val1
ret i64 %return
}
}

; CHECK: [[META0]] = !{i32 0}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,17 @@ define double @basic(<2 x ptr> %pointers) {
ret double %return
}

define double @should_preserve_metadata(<2 x ptr> %pointers) {
%pointer_to_double = getelementptr double, <2 x ptr> %pointers, i32 1, !any_metadata !{i32 0}

%ptr0 = extractelement <2 x ptr> %pointer_to_double, i32 0
%ptr1 = extractelement <2 x ptr> %pointer_to_double, i32 1
%val0 = load double, ptr %ptr0
%val1 = load double, ptr %ptr1
%return = fadd double %val0, %val1
ret double %return
}

define double @should_work_with_vector_of_indices(<2 x ptr> %pointers) {
%pointers_to_double = getelementptr double, <2 x ptr> %pointers, <2 x i32> <i32 0, i32 1>

Expand Down
Loading

0 comments on commit 6b857dc

Please sign in to comment.