diff --git a/IGC/AdaptorOCL/CMakeLists.txt b/IGC/AdaptorOCL/CMakeLists.txt index 35460c4c3d1d..be42fd52044b 100644 --- a/IGC/AdaptorOCL/CMakeLists.txt +++ b/IGC/AdaptorOCL/CMakeLists.txt @@ -133,6 +133,7 @@ set(IGC_BUILD__HDR__AdaptorOCL "${CMAKE_CURRENT_SOURCE_DIR}/ResolveConstExprCalls.h" "${CMAKE_CURRENT_SOURCE_DIR}/preprocess_spvir/PromoteBools.h" "${CMAKE_CURRENT_SOURCE_DIR}/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.h" + "${CMAKE_CURRENT_SOURCE_DIR}/preprocess_spvir/HandleSPIRVDecorations/CacheControlsHelper.h" "${CMAKE_CURRENT_SOURCE_DIR}/preprocess_spvir/PreprocessSPVIR.h" #"${IGC_BUILD__COMMON_COMPILER_DIR}/adapters/d3d10/API/USC_d3d10.h" diff --git a/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/CacheControlsHelper.h b/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/CacheControlsHelper.h new file mode 100644 index 000000000000..9401154e2750 --- /dev/null +++ b/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/CacheControlsHelper.h @@ -0,0 +1,145 @@ +/*========================== begin_copyright_notice ============================ + +Copyright (C) 2023 Intel Corporation + +SPDX-License-Identifier: MIT + +============================= end_copyright_notice ===========================*/ + +#pragma once + +#include "Compiler/CISACodeGen/getCacheOpts.h" + +namespace IGC +{ + enum class LoadCacheControl + { + Uncached = 0, + Cached = 1, + Streaming = 2, + InvalidateAfterRead = 3, + Invalid // This value represents invalid/unsupported cache control value + }; + + enum class StoreCacheControl + { + Uncached = 0, + WriteThrough = 1, + WriteBack = 2, + Streaming = 3, + Invalid // This value represents invalid/unsupported cache control value + }; + + template + using CacheControlMapTy = std::unordered_map, std::hash>; + + const CacheControlMapTy supportedStoreConfigs = + { + { LSC_L1UC_L3UC, {StoreCacheControl::Uncached, StoreCacheControl::Uncached } }, + { LSC_L1UC_L3C_WB, {StoreCacheControl::Uncached, StoreCacheControl::WriteBack } }, + { LSC_L1C_WT_L3UC, {StoreCacheControl::WriteThrough, StoreCacheControl::Uncached } }, + { LSC_L1C_WT_L3C_WB, {StoreCacheControl::WriteThrough, StoreCacheControl::WriteBack } }, + { LSC_L1S_L3UC, {StoreCacheControl::Streaming, StoreCacheControl::Uncached } }, + { LSC_L1S_L3C_WB, {StoreCacheControl::Streaming, StoreCacheControl::WriteBack } }, + { LSC_L1IAR_WB_L3C_WB, {StoreCacheControl::WriteBack, StoreCacheControl::WriteBack } } + }; + + const CacheControlMapTy supportedLoadConfigs = + { + { LSC_L1UC_L3UC, { LoadCacheControl::Uncached, LoadCacheControl::Uncached } }, + { LSC_L1UC_L3C_WB, { LoadCacheControl::Uncached, LoadCacheControl::Cached } }, + { LSC_L1C_WT_L3UC, { LoadCacheControl::Cached, LoadCacheControl::Uncached } }, + { LSC_L1C_WT_L3C_WB, { LoadCacheControl::Cached, LoadCacheControl::Cached } }, + { LSC_L1S_L3UC, { LoadCacheControl::Streaming, LoadCacheControl::Uncached } }, + { LSC_L1S_L3C_WB, { LoadCacheControl::Streaming, LoadCacheControl::Cached } }, + { LSC_L1IAR_WB_L3C_WB, { LoadCacheControl::InvalidateAfterRead, LoadCacheControl::Cached } }, + }; + + using CacheLevel = uint64_t; + + // This function expects MDNodes to be in the following form: + // MD = { opcode, cache_level, cache_control} + // ----------------------------------------------- + // !8 = !{i32 6442, i32 1, i32 0 } + // !9 = !{i32 6442, i32 3, i32 0 } + // !11 = !{i32 6442, i32 1, i32 0 } + // !12 = !{i32 6442, i32 3, i32 1 } + template + llvm::SmallDenseMap parseCacheControlsMD(llvm::SmallPtrSetImpl& MDNodes) + { + using namespace llvm; + auto getLiteral = [](const MDNode* node, const uint32_t index) + { + if (auto value = dyn_cast(node->getOperand(index))) + if (auto constantInt = dyn_cast(value->getValue())) + return std::optional(constantInt->getZExtValue()); + + return std::optional(); + }; + + SmallDenseMap cacheControls; + for (auto Node : MDNodes) + { + IGC_ASSERT(Node->getNumOperands() == 3); + IGC_ASSERT(getLiteral(Node, 0).value() == 6442 || + getLiteral(Node, 0).value() == 6443); + + auto cacheLevel = getLiteral(Node, 1); + auto cacheControl = getLiteral(Node, 2); + + IGC_ASSERT(cacheLevel && cacheControl); + cacheControls[cacheLevel.value()] = + static_cast(cacheControl.value()); + } + return cacheControls; + } + + template + std::optional getCacheControl(llvm::SmallDenseMap& cacheControls, CacheLevel level) + { + if (auto E = cacheControls.find(level); E != cacheControls.end()) + return E->second; + return {}; + } + + LSC_L1_L3_CC mapToLSCCacheControl(StoreCacheControl L1Control, StoreCacheControl L3Control) + { + for (auto& [LSCEnum, SPIRVEnum] : supportedStoreConfigs) + if (SPIRVEnum.first == L1Control && SPIRVEnum.second == L3Control) + return LSCEnum; + + return LSC_CC_INVALID; + } + + LSC_L1_L3_CC mapToLSCCacheControl(LoadCacheControl L1Control, LoadCacheControl L3Control) + { + for (auto& [LSCEnum, SPIRVEnum] : supportedLoadConfigs) + if (SPIRVEnum.first == L1Control && SPIRVEnum.second == L3Control) + return LSCEnum; + + return LSC_CC_INVALID; + } + + template + std::pair mapToSPIRVCacheControl(LSC_L1_L3_CC) = delete; + + template <> + std::pair mapToSPIRVCacheControl(LSC_L1_L3_CC LSCControl) + { + if (auto I = supportedLoadConfigs.find(LSCControl); I != supportedLoadConfigs.end()) + return I->second; + + IGC_ASSERT_MESSAGE(false, "Unsupported cache controls combination!"); + return { LoadCacheControl::Invalid, LoadCacheControl::Invalid }; + } + + template <> + std::pair mapToSPIRVCacheControl(LSC_L1_L3_CC LSCControl) + { + if (auto I = supportedStoreConfigs.find(LSCControl); I != supportedStoreConfigs.end()) + return I->second; + + IGC_ASSERT_MESSAGE(false, "Unsupported cache controls combination!"); + return { StoreCacheControl::Invalid, StoreCacheControl::Invalid }; + } +} diff --git a/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.cpp b/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.cpp index 2bf8a0299091..a21d9c25fba0 100644 --- a/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.cpp +++ b/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.cpp @@ -7,6 +7,7 @@ SPDX-License-Identifier: MIT ============================= end_copyright_notice ===========================*/ #include "HandleSpirvDecorationMetadata.h" +#include "CacheControlsHelper.h" #include "Compiler/IGCPassSupport.h" #include "Compiler/CISACodeGen/OpenCLKernelCodeGen.hpp" @@ -147,6 +148,12 @@ void HandleSpirvDecorationMetadata::visitLoadInst(LoadInst& I) { switch (DecorationId) { + // IDecCacheControlLoadINTEL + case 6442: + { + handleCacheControlINTEL(I, MDNodes); + break; + } default: continue; } } @@ -159,8 +166,75 @@ void HandleSpirvDecorationMetadata::visitStoreInst(StoreInst& I) { switch (DecorationId) { + // IDecCacheControlStoreINTEL + case 6443: + { + handleCacheControlINTEL(I, MDNodes); + break; + } default: continue; } } } +template +void HandleSpirvDecorationMetadata::handleCacheControlINTEL(Instruction& I, SmallPtrSetImpl& MDNodes) +{ + static_assert(std::is_same_v || std::is_same_v); + SmallDenseMap cacheControls = parseCacheControlsMD(MDNodes); + IGC_ASSERT(!cacheControls.empty()); + + // SPV_INTEL_cache_controls extension specification states the following: + // "Cache Level is an unsigned 32-bit integer telling the cache level to + // which the control applies. The value 0 indicates the cache level closest + // to the processing unit, the value 1 indicates the next furthest cache + // level, etc. If some cache level does not exist, the decoration is ignored." + // + // Therefore Cache Level equal to 0 maps to L1$ and Cache Level equal to 1 maps to L3$. + // Other Cache Level values are ignored. + auto L1CacheControl = getCacheControl(cacheControls, CacheLevel(0)); + auto L3CacheControl = getCacheControl(cacheControls, CacheLevel(1)); + + if (!L1CacheControl && !L3CacheControl) + { + // Early exit if there are no cache controls set for cache levels that are controllable + // by Intel GPUs. + return; + } + + LSC_L1_L3_CC defaultLSCCacheControls = static_cast( + std::is_same_v ? + m_pCtx->getModuleMetaData()->compOpt.LoadCacheDefault : + m_pCtx->getModuleMetaData()->compOpt.StoreCacheDefault); + + auto [L1Default, L3Default] = mapToSPIRVCacheControl(defaultLSCCacheControls); + + IGC_ASSERT(L1Default != T::Invalid && L3Default != T::Invalid); + + T newL1CacheControl = L1CacheControl ? L1CacheControl.value() : L1Default; + T newL3CacheControl = L3CacheControl ? L3CacheControl.value() : L3Default; + + LSC_L1_L3_CC newLSCCacheControl = + mapToLSCCacheControl(newL1CacheControl, newL3CacheControl); + + if (defaultLSCCacheControls == newLSCCacheControl) + { + // No need to set lsc.cache.ctrl metadata if requested cache controls are the same + // as default cache controls. + return; + } + + if (newLSCCacheControl != LSC_CC_INVALID) + { + MDNode* CacheCtrlNode = MDNode::get( + I.getContext(), + ConstantAsMetadata::get( + ConstantInt::get(Type::getInt32Ty(I.getContext()), newLSCCacheControl))); + I.setMetadata("lsc.cache.ctrl", CacheCtrlNode); + m_changed = true; + } + else + { + m_pCtx->EmitWarning("Unsupported cache controls configuration requested. Applying default configuration."); + } +} diff --git a/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.h b/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.h index be914f8f3cab..5b4797b78149 100644 --- a/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.h +++ b/IGC/AdaptorOCL/preprocess_spvir/HandleSPIRVDecorations/HandleSpirvDecorationMetadata.h @@ -58,6 +58,8 @@ namespace IGC void handleGlobalVariablesDecorations(); void handleHostAccessIntel(llvm::GlobalVariable& globalVariable, llvm::MDNode* node); + template + void handleCacheControlINTEL(llvm::Instruction& I, llvm::SmallPtrSetImpl& MDNodes); llvm::DenseMap> parseSPIRVDecorationsFromMD(llvm::Value* V); }; } diff --git a/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/kernel_arg.ll b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/kernel_arg.ll new file mode 100644 index 000000000000..e7ee09ac3c80 --- /dev/null +++ b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/kernel_arg.ll @@ -0,0 +1,54 @@ +;=========================== begin_copyright_notice ============================ +; +; Copyright (C) 2023 Intel Corporation +; +; SPDX-License-Identifier: MIT +; +;============================ end_copyright_notice ============================= + +; RUN: igc_opt %s -S -o - -serialize-igc-metadata -igc-handle-spirv-decoration-metadata | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024" +target triple = "spir64-unknown-unknown" + +define spir_kernel void @test(i32 addrspace(1)* %dummy, i32 addrspace(1)* %buffer) !spirv.ParameterDecorations !7 { +entry: + ; CHECK: store i32 0, i32 addrspace(1)* %buffer, align 4, !lsc.cache.ctrl [[S_CC:!.*]] + store i32 0, i32 addrspace(1)* %buffer, align 4 + ; CHECK: %0 = load i32, i32 addrspace(1)* %buffer, align 4, !lsc.cache.ctrl [[L_CC:!.*]] + %0 = load i32, i32 addrspace(1)* %buffer, align 4 + ret void +} + +!spirv.MemoryModel = !{!0} +!spirv.Source = !{!1} +!spirv.Generator = !{!2} +!IGCMetadata = !{!3} + +; CHECK: [[S_CC]] = !{i32 4} +; CHECK: [[L_CC]] = !{i32 2} + +; Above literals represent the following enums: +; 1 - LSC_L1UC_L3UC +; 2 - LSC_L1UC_L3C_WB +; 3 - LSC_L1C_WT_L3UC +; 4 - LSC_L1C_WT_L3C_WB +; 5 - LSC_L1S_L3UC +; 6 - LSC_L1S_L3C_WB +; 7 - LSC_L1IAR_WB_L3C_WB + +!0 = !{i32 2, i32 2} +!1 = !{i32 3, i32 102000} +!2 = !{i16 6, i16 14} +!3 = !{!"ModuleMD", !4} +!4 = !{!"compOpt", !5, !6} +; Below values represent the default cache controls that in real compilation are passed +; from OCL/L0 Runtime as internal options and may vary depending on the target platform. +; These are the default values for PVC: +!5 = !{!"LoadCacheDefault", i32 4} ; LSC_L1C_WT_L3C_WB - loads: L1 cached, L3 cached +!6 = !{!"StoreCacheDefault", i32 2} ; LSC_L1UC_L3C_WB - stores: L1 uncached, L3 write-back +!7 = !{!8, !9} +!8 = !{} +!9 = !{!10, !11} +!10 = !{i32 6442, i32 0, i32 0} ; {CacheControlStoreINTEL, CacheLevel=0, WriteThrough} +!11 = !{i32 6443, i32 0, i32 1} ; {CacheControlLoadINTEL, CacheLevel=0, Uncached} diff --git a/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/loads.ll b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/loads.ll new file mode 100644 index 000000000000..0ed751643fd3 --- /dev/null +++ b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/loads.ll @@ -0,0 +1,93 @@ +;=========================== begin_copyright_notice ============================ +; +; Copyright (C) 2023 Intel Corporation +; +; SPDX-License-Identifier: MIT +; +;============================ end_copyright_notice ============================= + +; RUN: igc_opt %s -S -o - -serialize-igc-metadata -igc-handle-spirv-decoration-metadata | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-n8:16:32" +target triple = "spir64-unknown-unknown" + +define spir_kernel void @test(i32 addrspace(1)* %buffer) { +entry: + %arrayidx0 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !7 + ; CHECK: %l0 = load i32, i32 addrspace(1)* %arrayidx0, align 4, !lsc.cache.ctrl [[L0_CC:!.*]] + %l0 = load i32, i32 addrspace(1)* %arrayidx0, align 4 + %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !10 + ; CHECK: %l1 = load i32, i32 addrspace(1)* %arrayidx1, align 4, !lsc.cache.ctrl [[L1_CC:!.*]] + %l1 = load i32, i32 addrspace(1)* %arrayidx1, align 4 + %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !13 + ; CHECK: %l2 = load i32, i32 addrspace(1)* %arrayidx2, align 4, !lsc.cache.ctrl [[L2_CC:!.*]] + %l2 = load i32, i32 addrspace(1)* %arrayidx2, align 4 + %arrayidx3 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !16 + ; COM: cached controls specified by !16 metadata are the same as default cache controls, so not + ; expecting insertion of !lsc.cache.ctrl metadata + ; CHECK-NOT: load i32, i32 addrspace(1)* %arrayidx3, align 4, !lsc.cache.ctrl + %l3 = load i32, i32 addrspace(1)* %arrayidx3, align 4 + %arrayidx4 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !19 + ; CHECK: %l4 = load i32, i32 addrspace(1)* %arrayidx4, align 4, !lsc.cache.ctrl [[L4_CC:!.*]] + %l4 = load i32, i32 addrspace(1)* %arrayidx4, align 4 + %arrayidx5 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !22 + ; CHECK: %l5 = load i32, i32 addrspace(1)* %arrayidx5, align 4, !lsc.cache.ctrl [[L5_CC:!.*]] + %l5 = load i32, i32 addrspace(1)* %arrayidx5, align 4 + %arrayidx6 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !25 + ; CHECK: %l6 = load i32, i32 addrspace(1)* %arrayidx6, align 4, !lsc.cache.ctrl [[L6_CC:!.*]] + %l6 = load i32, i32 addrspace(1)* %arrayidx6, align 4 + ret void +} + +!spirv.MemoryModel = !{!0} +!spirv.Source = !{!1} +!spirv.Generator = !{!2} +!IGCMetadata = !{!3} + +; CHECK: [[L0_CC]] = !{i32 1} +; CHECK: [[L1_CC]] = !{i32 2} +; CHECK: [[L2_CC]] = !{i32 3} +; CHECK: [[L4_CC]] = !{i32 5} +; CHECK: [[L5_CC]] = !{i32 6} +; CHECK: [[L6_CC]] = !{i32 7} + +; Above literals represent the following enums: +; 1 - LSC_L1UC_L3UC +; 2 - LSC_L1UC_L3C_WB +; 3 - LSC_L1C_WT_L3UC +; 4 - LSC_L1C_WT_L3C_WB +; 5 - LSC_L1S_L3UC +; 6 - LSC_L1S_L3C_WB +; 7 - LSC_L1IAR_WB_L3C_WB + +!0 = !{i32 2, i32 2} +!1 = !{i32 3, i32 102000} +!2 = !{i16 6, i16 14} +!3 = !{!"ModuleMD", !4} +!4 = !{!"compOpt", !5, !6} +; Below values represent the default cache controls that in real compilation are passed +; from OCL/L0 Runtime as internal options and may vary depending on the target platform. +; These are the default values for PVC: +!5 = !{!"LoadCacheDefault", i32 4} ; LSC_L1C_WT_L3C_WB - loads: L1 cached, L3 cached +!6 = !{!"StoreCacheDefault", i32 2} ; LSC_L1UC_L3C_WB - stores: L1 uncached, L3 write-back +!7 = !{!8, !9} +!8 = !{i32 6442, i32 0, i32 0} ; {CacheControlLoadINTEL, CacheLevel=0, Uncached} +!9 = !{i32 6442, i32 1, i32 0} ; {CacheControlLoadINTEL, CacheLevel=1, Uncached} +!10 = !{!11, !12} +!11 = !{i32 6442, i32 0, i32 0} ; {CacheControlLoadINTEL, CacheLevel=0, Uncached} +!12 = !{i32 6442, i32 1, i32 1} ; {CacheControlLoadINTEL, CacheLevel=1, Cached} +!13 = !{!14, !15} +!14 = !{i32 6442, i32 0, i32 1} ; {CacheControlLoadINTEL, CacheLevel=0, Cached} +!15 = !{i32 6442, i32 1, i32 0} ; {CacheControlLoadINTEL, CacheLevel=1, Uncached} +!16 = !{!17, !18} +!17 = !{i32 6442, i32 0, i32 1} ; {CacheControlLoadINTEL, CacheLevel=0, Cached} +!18 = !{i32 6442, i32 1, i32 1} ; {CacheControlLoadINTEL, CacheLevel=1, Cached} +!19 = !{!20, !21} +!20 = !{i32 6442, i32 0, i32 2} ; {CacheControlLoadINTEL, CacheLevel=0, Streaming} +!21 = !{i32 6442, i32 1, i32 0} ; {CacheControlLoadINTEL, CacheLevel=1, Uncached} +!22 = !{!23, !24} +!23 = !{i32 6442, i32 0, i32 2} ; {CacheControlLoadINTEL, CacheLevel=0, Streaming} +!24 = !{i32 6442, i32 1, i32 1} ; {CacheControlLoadINTEL, CacheLevel=1, Cached} +!25 = !{!26, !27} +!26 = !{i32 6442, i32 0, i32 3} ; {CacheControlLoadINTEL, CacheLevel=0, InvalidateAfterRead} +!27 = !{i32 6442, i32 1, i32 1} ; {CacheControlLoadINTEL, CacheLevel=1, Cached} diff --git a/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/partial-cache-controls-loads.ll b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/partial-cache-controls-loads.ll new file mode 100644 index 000000000000..0fc69714b98a --- /dev/null +++ b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/partial-cache-controls-loads.ll @@ -0,0 +1,67 @@ +;=========================== begin_copyright_notice ============================ +; +; Copyright (C) 2023 Intel Corporation +; +; SPDX-License-Identifier: MIT +; +;============================ end_copyright_notice ============================= + +; RUN: igc_opt %s -S -o - -serialize-igc-metadata -igc-handle-spirv-decoration-metadata | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-n8:16:32" +target triple = "spir64-unknown-unknown" + +define spir_func void @test(i32 addrspace(1)* %buffer) { +entry: + ; Test changing load default { L1 Cached, L3 Cached } -> { L1 Uncached, L3 Cached } + %arrayidx0 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !7 + ; CHECK: %l1 = load i32, i32 addrspace(1)* %arrayidx0, align 4, !lsc.cache.ctrl [[L0_CC:!.*]] + %l1 = load i32, i32 addrspace(1)* %arrayidx0, align 4 + + ; Test changing load default { L1 Cached, L3 Cached } -> { L1 Streaming, L3 Cached } + %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !9 + ; CHECK: %l2 = load i32, i32 addrspace(1)* %arrayidx1, align 4, !lsc.cache.ctrl [[L1_CC:!.*]] + %l2 = load i32, i32 addrspace(1)* %arrayidx1, align 4 + + ; Test changing load default { L1 Cached, L3 Cached } -> { L1 Cached, L3 Uncached } + %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !11 + ; CHECK: %l0 = load i32, i32 addrspace(1)* %arrayidx2, align 4, !lsc.cache.ctrl [[L2_CC:!.*]] + %l0 = load i32, i32 addrspace(1)* %arrayidx2, align 4 + + ret void +} + +!spirv.MemoryModel = !{!0} +!spirv.Source = !{!1} +!spirv.Generator = !{!2} +!IGCMetadata = !{!3} + +; CHECK: [[L0_CC]] = !{i32 2} +; CHECK: [[L1_CC]] = !{i32 6} +; CHECK: [[L2_CC]] = !{i32 3} + +; Above literals represent the following enums: +; 1 - LSC_L1UC_L3UC +; 2 - LSC_L1UC_L3C_WB +; 3 - LSC_L1C_WT_L3UC +; 4 - LSC_L1C_WT_L3C_WB +; 5 - LSC_L1S_L3UC +; 6 - LSC_L1S_L3C_WB +; 7 - LSC_L1IAR_WB_L3C_WB + +!0 = !{i32 2, i32 2} +!1 = !{i32 3, i32 102000} +!2 = !{i16 6, i16 14} +!3 = !{!"ModuleMD", !4} +!4 = !{!"compOpt", !5, !6} +; Below values represent the default cache controls that in real compilation are passed +; from OCL/L0 Runtime as internal options and may vary depending on the target platform. +; These are the default values for PVC: +!5 = !{!"LoadCacheDefault", i32 4} ; LSC_L1C_WT_L3C_WB - loads: L1 cached, L3 cached +!6 = !{!"StoreCacheDefault", i32 2} ; LSC_L1UC_L3C_WB - stores: L1 uncached, L3 write-back +!7 = !{!8} +!8 = !{i32 6442, i32 0, i32 0} ; {CacheControlLoadINTEL, CacheLevel=0, Uncached} +!9 = !{!10} +!10 = !{i32 6442, i32 0, i32 2} ; {CacheControlLoadINTEL, CacheLevel=0, Streaming} +!11= !{!12} +!12 = !{i32 6442, i32 1, i32 0} ; {CacheControlLoadINTEL, CacheLevel=1, Uncached} diff --git a/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/partial-cache-controls-stores.ll b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/partial-cache-controls-stores.ll new file mode 100644 index 000000000000..69b64f466a87 --- /dev/null +++ b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/partial-cache-controls-stores.ll @@ -0,0 +1,75 @@ +;=========================== begin_copyright_notice ============================ +; +; Copyright (C) 2023 Intel Corporation +; +; SPDX-License-Identifier: MIT +; +;============================ end_copyright_notice ============================= + +; RUN: igc_opt %s -S -o - -serialize-igc-metadata -igc-handle-spirv-decoration-metadata | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-n8:16:32" +target triple = "spir64-unknown-unknown" + +define spir_func void @test(i32 addrspace(1)* %buffer) { + ; Test changing store default { L1 Uncached, L3 WriteBack } -> { L1 WriteThrough, L3 WriteBack } + %arrayidx0 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !7 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx0, align 4, !lsc.cache.ctrl [[S0_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx0, align 4 + + ; Test changing store default { L1 Uncached, L3 WriteBack } -> { L1 Streaming, L3 WriteBack } + %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !9 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx1, align 4, !lsc.cache.ctrl [[S1_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx1, align 4 + + ; Test changing store default { L1 Uncached, L3 WriteBack } -> { L1 WriteBack, L3 WriteBack } + %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !11 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx2, align 4, !lsc.cache.ctrl [[S2_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx2, align 4 + + ; Test changing store default { L1 Uncached, L3 WriteBack } -> { L1 Uncached, L3 Uncached } + %arrayidx3 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !13 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx3, align 4, !lsc.cache.ctrl [[S3_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx3, align 4 + + ret void +} + +!spirv.MemoryModel = !{!0} +!spirv.Source = !{!1} +!spirv.Generator = !{!2} +!IGCMetadata = !{!3} + +; CHECK: [[S0_CC]] = !{i32 4} +; CHECK: [[S1_CC]] = !{i32 6} +; CHECK: [[S2_CC]] = !{i32 7} +; CHECK: [[S3_CC]] = !{i32 1} + +; Above literals represent the following enums: +; 1 - LSC_L1UC_L3UC +; 2 - LSC_L1UC_L3C_WB +; 3 - LSC_L1C_WT_L3UC +; 4 - LSC_L1C_WT_L3C_WB +; 5 - LSC_L1S_L3UC +; 6 - LSC_L1S_L3C_WB +; 7 - LSC_L1IAR_WB_L3C_WB + + +!0 = !{i32 2, i32 2} +!1 = !{i32 3, i32 102000} +!2 = !{i16 6, i16 14} +!3 = !{!"ModuleMD", !4} +!4 = !{!"compOpt", !5, !6} +; Below values represent the default cache controls that in real compilation are passed +; from OCL/L0 Runtime as internal options and may vary depending on the target platform. +; These are the default values for PVC: +!5 = !{!"LoadCacheDefault", i32 4} ; LSC_L1C_WT_L3C_WB - loads: L1 cached, L3 cached +!6 = !{!"StoreCacheDefault", i32 2} ; LSC_L1UC_L3C_WB - stores: L1 uncached, L3 write-back +!7 = !{!8} +!8 = !{i32 6443, i32 0, i32 1} ; {CacheControlStoreINTEL, CacheLevel=0, WriteThrough} +!9 = !{!10} +!10 = !{i32 6443, i32 0, i32 3} ; {CacheControlStoreINTEL, CacheLevel=0, Streaming} +!11 = !{!12} +!12 = !{i32 6443, i32 0, i32 2} ; {CacheControlStoreINTEL, CacheLevel=0, WriteBack} +!13 = !{!14} +!14 = !{i32 6443, i32 1, i32 0} ; {CacheControlStoreINTEL, CacheLevel=1, Uncached} diff --git a/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/stores.ll b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/stores.ll new file mode 100644 index 000000000000..d0b568ed4c92 --- /dev/null +++ b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/stores.ll @@ -0,0 +1,93 @@ +;=========================== begin_copyright_notice ============================ +; +; Copyright (C) 2023 Intel Corporation +; +; SPDX-License-Identifier: MIT +; +;============================ end_copyright_notice ============================= + +; RUN: igc_opt %s -S -o - -serialize-igc-metadata -igc-handle-spirv-decoration-metadata | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-n8:16:32" +target triple = "spir64-unknown-unknown" + +define spir_kernel void @test(i32 addrspace(1)* %buffer) { +entry: + %arrayidx0 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !7 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx0, align 4, !lsc.cache.ctrl [[S0_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx0, align 4 + %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !10 + ; COM: cached controls specified by !10 metadata are the same as default cache controls, so not + ; expecting insertion of !lsc.cache.ctrl metadata + ; CHECK-NOT: store i32 0, i32 addrspace(1)* %arrayidx1, align 4, !lsc.cache.ctrl + store i32 0, i32 addrspace(1)* %arrayidx1, align 4 + %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !13 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx2, align 4, !lsc.cache.ctrl [[S2_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx2, align 4 + %arrayidx3 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !16 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx3, align 4, !lsc.cache.ctrl [[S3_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx3, align 4 + %arrayidx4 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !19 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx4, align 4, !lsc.cache.ctrl [[S4_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx4, align 4 + %arrayidx5 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !22 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx5, align 4, !lsc.cache.ctrl [[S5_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx5, align 4 + %arrayidx6 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !25 + ; CHECK: store i32 0, i32 addrspace(1)* %arrayidx6, align 4, !lsc.cache.ctrl [[S6_CC:!.*]] + store i32 0, i32 addrspace(1)* %arrayidx6, align 4 + ret void +} + +!spirv.MemoryModel = !{!0} +!spirv.Source = !{!1} +!spirv.Generator = !{!2} +!IGCMetadata = !{!3} + +; CHECK: [[S0_CC]] = !{i32 1} +; CHECK: [[S2_CC]] = !{i32 3} +; CHECK: [[S3_CC]] = !{i32 4} +; CHECK: [[S4_CC]] = !{i32 5} +; CHECK: [[S5_CC]] = !{i32 6} +; CHECK: [[S6_CC]] = !{i32 7} + +; Above literals represent the following enums: +; 1 - LSC_L1UC_L3UC +; 2 - LSC_L1UC_L3C_WB +; 3 - LSC_L1C_WT_L3UC +; 4 - LSC_L1C_WT_L3C_WB +; 5 - LSC_L1S_L3UC +; 6 - LSC_L1S_L3C_WB +; 7 - LSC_L1IAR_WB_L3C_WB + +!0 = !{i32 2, i32 2} +!1 = !{i32 3, i32 102000} +!2 = !{i16 6, i16 14} +!3 = !{!"ModuleMD", !4} +!4 = !{!"compOpt", !5, !6} +; Below values represent the default cache controls that in real compilation are passed +; from OCL/L0 Runtime as internal options and may vary depending on the target platform. +; These are the default values for PVC: +!5 = !{!"LoadCacheDefault", i32 4} ; LSC_L1C_WT_L3C_WB - loads: L1 cached, L3 cached +!6 = !{!"StoreCacheDefault", i32 2} ; LSC_L1UC_L3C_WB - stores: L1 uncached, L3 write-back +!7 = !{!8, !9} +!8 = !{i32 6443, i32 0, i32 0} ; {CacheControlStoreINTEL, CacheLevel=0, Uncached} +!9 = !{i32 6443, i32 1, i32 0} ; {CacheControlStoreINTEL, CacheLevel=1, Uncached} +!10 = !{!11, !12} +!11 = !{i32 6443, i32 0, i32 0} ; {CacheControlStoreINTEL, CacheLevel=0, Uncached} +!12 = !{i32 6443, i32 1, i32 2} ; {CacheControlStoreINTEL, CacheLevel=1, WriteBack} +!13 = !{!14, !15} +!14 = !{i32 6443, i32 0, i32 1} ; {CacheControlStoreINTEL, CacheLevel=0, WriteThrough} +!15 = !{i32 6443, i32 1, i32 0} ; {CacheControlStoreINTEL, CacheLevel=1, Uncached} +!16 = !{!17, !18} +!17 = !{i32 6443, i32 0, i32 1} ; {CacheControlStoreINTEL, CacheLevel=0, WriteThrough} +!18 = !{i32 6443, i32 1, i32 2} ; {CacheControlStoreINTEL, CacheLevel=1, WriteBack} +!19 = !{!20, !21} +!20 = !{i32 6443, i32 0, i32 3} ; {CacheControlStoreINTEL, CacheLevel=0, Streaming} +!21 = !{i32 6443, i32 1, i32 0} ; {CacheControlStoreINTEL, CacheLevel=1, Uncached} +!22 = !{!23, !24} +!23 = !{i32 6443, i32 0, i32 3} ; {CacheControlStoreINTEL, CacheLevel=0, Streaming} +!24 = !{i32 6443, i32 1, i32 2} ; {CacheControlStoreINTEL, CacheLevel=1, WriteBack} +!25 = !{!26, !27} +!26 = !{i32 6443, i32 0, i32 2} ; {CacheControlStoreINTEL, CacheLevel=0, WriteBack} +!27 = !{i32 6443, i32 1, i32 2} ; {CacheControlStoreINTEL, CacheLevel=1, WriteBack} diff --git a/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/unsupported-cache-config.ll b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/unsupported-cache-config.ll new file mode 100644 index 000000000000..dc7f56a24d2a --- /dev/null +++ b/IGC/Compiler/tests/HandleSpirvDecorationsMetadata/CacheControls/unsupported-cache-config.ll @@ -0,0 +1,50 @@ +;=========================== begin_copyright_notice ============================ +; +; Copyright (C) 2023 Intel Corporation +; +; SPDX-License-Identifier: MIT +; +;============================ end_copyright_notice ============================= + +; RUN: igc_opt %s -S -o - -serialize-igc-metadata -igc-handle-spirv-decoration-metadata --igc-error-check < %s 2>&1 | FileCheck %s + +; The below test tries to set { L1 cached, L3 streaming } cache controls configuration that is not supported on PVC. + +; Here are all cache controls combinations that are supported on PVC: +; LSC_L1UC_L3UC +; LSC_L1UC_L3C_WB +; LSC_L1C_WT_L3UC +; LSC_L1C_WT_L3C_WB +; LSC_L1S_L3UC +; LSC_L1S_L3C_WB +; LSC_L1IAR_WB_L3C_WB + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-n8:16:32" +target triple = "spir64-unknown-unknown" + +define spir_func void @test(i32 addrspace(1)* %buffer) { +entry: + %arrayidx0 = getelementptr inbounds i32, i32 addrspace(1)* %buffer, i64 1, !spirv.Decorations !7 + ; CHECK: warning: Unsupported cache controls configuration requested. Applying default configuration. + %l0 = load i32, i32 addrspace(1)* %arrayidx0, align 4 + + ret void +} + +!spirv.MemoryModel = !{!0} +!spirv.Source = !{!1} +!spirv.Generator = !{!2} +!IGCMetadata = !{!3} + +!0 = !{i32 2, i32 2} +!1 = !{i32 3, i32 102000} +!2 = !{i16 6, i16 14} +!3 = !{!"ModuleMD", !4} +!4 = !{!"compOpt", !5, !6} +; Below values represent the default cache controls that in real compilation are passed +; from OCL/L0 Runtime as internal options and may vary depending on the target platform. +; These are the default values for PVC: +!5 = !{!"LoadCacheDefault", i32 4} ; LSC_L1C_WT_L3C_WB - loads: L1 cached, L3 cached +!6 = !{!"StoreCacheDefault", i32 2} ; LSC_L1UC_L3C_WB - stores: L1 uncached, L3 write-back +!7 = !{!8} +!8 = !{i32 6442, i32 1, i32 2}