Promotion from AMD internal branch for 2023.Q3.2

cmake/CompilerFlags.cmake

@@ -31,6 +31,8 @@ function(set_compiler_options PROJECT_NAME ENABLE_WERROR)
             -Wunused-function
             -Werror=unused-function
             -Werror=unused-result  # Error out on unused results of functions marked with LLPC_NODISCARD
+            -ffunction-sections
+            -fdata-sections
         )
 
         target_compile_options("${PROJECT_NAME}" PRIVATE $<$<COMPILE_LANGUAGE:CXX>:

include/vkgcDefs.h

@@ -49,7 +49,7 @@
 #define LLPC_INTERFACE_MAJOR_VERSION 66
 
 /// LLPC minor interface version.
-#define LLPC_INTERFACE_MINOR_VERSION 0
+#define LLPC_INTERFACE_MINOR_VERSION 1
 
 #ifndef LLPC_CLIENT_INTERFACE_MAJOR_VERSION
 #error LLPC client version is not defined
@@ -84,8 +84,13 @@
 //  |     66.0 | Rename noContract in PipelineShaderOptions to noContractOpDot                                         |
 //  |     65.1 | Add disableSampleMask to PipelineOptions                                                              |
 //  |     65.0 | Remove updateDescInElf                                                                                |
+//  |     64.2 | Add dynamicSampleInfo to GraphicsPipelineBuildInfo::rsState                                           |
+//  |     64.1 | Add disableTruncCoordForGather to PipelineOptions.                                                    |
 //  |     64.0 | Add enableColorExportShader to GraphicsPipelineBuildInfo.                                             |
-//  |     63.0 | Add Atomic Counter, its default descriptor and map its concreteType to Buffer.                        |
+//  |     63.3 | Add TesssellationLevel to iaState                                                                     |
+//  |     63.2 | Add vertex64BitsAttribSingleLoc to PipelineOptions                                                    |
+//  |     63.1 | Add forceDisableStreamOut and forceEnablePrimStats to ApiXfbOutData                                   |
+//  |     63.0 | Add Atomic Counter, its default descriptor and map its concertType to Buffer.                         |
 //  |     62.1 | Add ApiXfbOutData GraphicsPipelineBuildInfo                                                           |
 //  |     62.0 | Default to the compiler getting the GPURT library directly, and move shader library info into RtState |
 //  |     61.16| Add replaceSetWithResourceType to PipelineOptions                                                     |
@@ -459,9 +464,6 @@ struct ResourceMappingNode {
       unsigned strideInDwords; ///< Stride of elements in a descriptor array (used for mutable descriptors)
                                ///  a stride of zero will use the type of the node to determine the stride
 #endif
-      unsigned reserv0;
-      unsigned reserv1;
-      unsigned reserv2;
     } srdRange;
     /// Info for hierarchical nodes (DescriptorTableVaPtr)
     struct {
@@ -588,6 +590,10 @@ struct PipelineOptions {
   bool replaceSetWithResourceType;        ///< For OGL only, replace 'set' with resource type during spirv translate
   bool disableSampleMask;                 ///< For OGL only, disabled if framebuffer doesn't attach multisample texture
   bool buildResourcesDataForShaderModule; ///< For OGL only, build resources usage data while building shader module
+  bool disableTruncCoordForGather;        ///< If set, trunc_coord of sampler srd is disabled for gather4
+  bool enableCombinedTexture;             ///< For OGL only, use the 'set' for DescriptorCombinedTexture
+                                          ///< for sampled images and samplers
+  bool vertex64BitsAttribSingleLoc;       ///< For OGL only, dvec3/dvec4 vertex attrib only consumes 1 location.
 };
 
 /// Prototype of allocator for output data buffer, used in shader-specific operations.
@@ -1201,8 +1207,16 @@ struct XfbOutInfo {
 
 /// Represents the transform feedback data filled by API interface
 struct ApiXfbOutData {
-  XfbOutInfo *pXfbOutInfos; ///< An array of XfbOutInfo iterms
-  unsigned numXfbOutInfo;   ///< Count of XfbOutInfo iterms
+  XfbOutInfo *pXfbOutInfos;   ///< An array of XfbOutInfo iterms
+  unsigned numXfbOutInfo;     ///< Count of XfbOutInfo iterms
+  bool forceDisableStreamOut; ///< Force to disable stream out XFB outputs
+  bool forceEnablePrimStats;  ///< Force to enable counting generated primitives
+};
+
+/// Represents the tessellation level passed from driver API
+struct TessellationLevel {
+  float inner[2]; ///< Inner tessellation level
+  float outer[4]; ///< Outer tessellation level
 };
 
 /// Represents info to build a graphics pipeline.
@@ -1238,6 +1252,7 @@ struct GraphicsPipelineBuildInfo {
     bool enableMultiView;          ///< Whether to enable multi-view support
     bool useVertexBufferDescArray; ///< Whether vertex buffer descriptors are in a descriptor array binding instead of
                                    ///< the VertexBufferTable
+    TessellationLevel *tessLevel;  ///< Tessellation level passed from driver
   } iaState;                       ///< Input-assembly state
 
   struct {
@@ -1257,6 +1272,7 @@ struct GraphicsPipelineBuildInfo {
     unsigned samplePatternIdx;    ///< Index into the currently bound MSAA sample pattern table that
                                   ///  matches the sample pattern used by the rasterizer when rendering
                                   ///  with this pipeline.
+    unsigned dynamicSampleInfo;   ///< Dynamic sampling is enabled.
     unsigned rasterStream;        ///< Which vertex stream to rasterize
     VkProvokingVertexModeEXT provokingVertexMode; ///< Specifies which vertex of a primitive is the _provoking
                                                   ///  vertex_, this impacts which vertex's "flat" VS outputs

lgc/CMakeLists.txt

@@ -227,6 +227,7 @@ target_sources(LLVMlgc PRIVATE
     patch/PatchReadFirstLane.cpp
     patch/PatchResourceCollect.cpp
     patch/PatchSetupTargetFeatures.cpp
+    patch/TcsPassthroughShader.cpp
     patch/PatchInitializeWorkgroupMemory.cpp
     patch/PatchWorkarounds.cpp
     patch/ShaderInputs.cpp

lgc/builder/ImageBuilder.cpp

@@ -493,7 +493,10 @@ Value *BuilderImpl::CreateImageLoad(Type *resultTy, unsigned dim, unsigned flags
 
     // Get the intrinsic ID from the load intrinsic ID table and call it.
     auto table = mipLevel ? &ImageLoadMipIntrinsicTable[0] : &ImageLoadIntrinsicTable[0];
-    imageInst = CreateIntrinsic(table[dim], {intrinsicDataTy, coords[0]->getType()}, args, nullptr, instName);
+
+    // Rectangle image uses the same Intrinsic ID with 2D image.
+    Intrinsic::ID intrinsicId = (dim == DimRect) ? table[Dim2D] : table[dim];
+    imageInst = CreateIntrinsic(intrinsicId, {intrinsicDataTy, coords[0]->getType()}, args, nullptr, instName);
   } else {
     // Texel buffer descriptor. Use the buffer instruction.
     imageDescArgIndex = args.size();
@@ -683,7 +686,10 @@ Value *BuilderImpl::CreateImageStore(Value *texel, unsigned dim, unsigned flags,
 
     // Get the intrinsic ID from the store intrinsic ID table and call it.
     auto table = mipLevel ? &ImageStoreMipIntrinsicTable[0] : &ImageStoreIntrinsicTable[0];
-    imageStore = CreateIntrinsic(table[dim], {texelTy, coords[0]->getType()}, args, nullptr, instName);
+
+    // Rectangle image uses the same Intrinsic ID with 2D image.
+    Intrinsic::ID intrinsicId = (dim == DimRect) ? table[Dim2D] : table[dim];
+    imageStore = CreateIntrinsic(intrinsicId, {texelTy, coords[0]->getType()}, args, nullptr, instName);
   } else {
     // Texel buffer descriptor. Use the buffer instruction.
     // First widen texel to vec4 if necessary.
@@ -856,6 +862,10 @@ Value *BuilderImpl::CreateImageGather(Type *resultTy, unsigned dim, unsigned fla
   // Only the first 4 dwords are sampler descriptor, we need to extract these values under any condition
   samplerDesc = CreateShuffleVector(samplerDesc, samplerDesc, ArrayRef<int>{0, 1, 2, 3});
 
+  if (m_pipelineState->getOptions().disableTruncCoordForGather) {
+    samplerDesc = modifySamplerDescForGather(samplerDesc);
+  }
+
   Value *result = nullptr;
   Value *addrOffset = address[ImageAddressIdxOffset];
   if (addrOffset && isa<ArrayType>(addrOffset->getType())) {
@@ -1149,7 +1159,7 @@ Value *BuilderImpl::CreateImageSampleGather(Type *resultTy, unsigned dim, unsign
   args.push_back(samplerDesc);
 
   // i32 Unorm
-  args.push_back(getInt1(false));
+  args.push_back(getInt1(dim == DimRect));
 
   // i32 tfe/lwe bits
   bool tfe = isa<StructType>(resultTy);
@@ -1174,7 +1184,9 @@ Value *BuilderImpl::CreateImageSampleGather(Type *resultTy, unsigned dim, unsign
     if (table->matchMask == addressMask)
       break;
   }
-  Intrinsic::ID intrinsicId = table->ids[dim];
+
+  // Rectangle texture uses the same Intrinsic ID with 2D texture.
+  Intrinsic::ID intrinsicId = (dim == DimRect) ? table->ids[Dim2D] : table->ids[dim];
 
   // Create the intrinsic.
   Instruction *imageOp = CreateIntrinsic(intrinsicId, overloadTys, args, nullptr, instName);
@@ -1282,7 +1294,9 @@ Value *BuilderImpl::CreateImageAtomicCommon(unsigned atomicOp, unsigned dim, uns
     args.push_back(getInt32(0));
 
     // Get the intrinsic ID from the load intrinsic ID table, and create the intrinsic.
-    Intrinsic::ID intrinsicId = ImageAtomicIntrinsicTable[atomicOp][dim];
+    // Rectangle image uses the same Intrinsic ID with 2D image.
+    Intrinsic::ID intrinsicId =
+        (dim == DimRect) ? ImageAtomicIntrinsicTable[atomicOp][Dim2D] : ImageAtomicIntrinsicTable[atomicOp][dim];
     atomicInst = CreateIntrinsic(intrinsicId, {inputValue->getType(), coord->getType()->getScalarType()}, args, nullptr,
                                  instName);
   } else {
@@ -1626,8 +1640,6 @@ unsigned BuilderImpl::prepareCoordinate(unsigned dim, Value *coord, Value *proje
     assert(getImageNumCoords(dim) == 1);
     outCoords.push_back(coord);
   } else {
-    assert(getImageNumCoords(dim) == cast<FixedVectorType>(coordTy)->getNumElements());
-
     // Push the components.
     for (unsigned i = 0; i != getImageNumCoords(dim); ++i)
       outCoords.push_back(CreateExtractElement(coord, i));
@@ -2061,3 +2073,17 @@ void BuilderImpl::enforceReadFirstLane(Instruction *imageInst, unsigned descIdx)
   }
   imageInst->setOperand(descIdx, newDesc);
 }
+
+// =====================================================================================================================
+// Modify sampler descriptor to force set trunc_coord as 0 for gather4 instruction.
+//
+// @param imageDesc : Original sampler descriptor
+// @returns Sampler descriptor, modified if necessary
+Value *BuilderImpl::modifySamplerDescForGather(Value *samplerDesc) {
+  // Need to clear the trunc_coord bit for gather4, which is bit 27 of dword 0.
+  Value *dword0 = CreateExtractElement(samplerDesc, uint64_t(0));
+  dword0 = CreateAnd(dword0, getInt32(0xF7FFFFFF));
+  samplerDesc = CreateInsertElement(samplerDesc, dword0, uint64_t(0));
+
+  return samplerDesc;
+}

lgc/elfLinker/RelocHandler.cpp

@@ -181,14 +181,6 @@ bool RelocHandler::getValue(StringRef name, uint64_t &value) {
     }
   }
 
-  if (name == reloc::NumSamples) {
-    value = m_pipelineState->getRasterizerState().numSamples;
-    return true;
-  }
-  if (name == reloc::SamplePatternIdx) {
-    value = m_pipelineState->getRasterizerState().samplePatternIdx;
-    return true;
-  }
   if (name == reloc::DeviceIdx) {
     value = m_pipelineState->getDeviceIndex();
     return true;

lgc/include/lgc/builder/BuilderImpl.h

@@ -441,6 +441,9 @@ class BuilderImpl : public BuilderDefs {
   // Enforce readfirstlane on the image or sampler descriptors
   void enforceReadFirstLane(llvm::Instruction *imageInst, unsigned descIdx);
 
+  // Modify sampler descriptor to force set trunc_coord as 0 for gather4 instruction.
+  llvm::Value *modifySamplerDescForGather(llvm::Value *samplerDesc);
+
   enum ImgDataFormat {
     IMG_DATA_FORMAT_32 = 4,
     IMG_DATA_FORMAT_8_8_8_8 = 10,

lgc/include/lgc/patch/TcsPassthroughShader.h

@@ -0,0 +1,54 @@
+/*
+ ***********************************************************************************************************************
+ *
+ *  Copyright (c) 2021-2023 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *  SOFTWARE.
+ *
+ **********************************************************************************************************************/
+/**
+ ***********************************************************************************************************************
+ * @file  TcsPassthroughShader.h
+ * @brief LLPC header file: contains declaration of class lgc::TcsPassthroughShader.
+ ***********************************************************************************************************************
+ */
+#pragma once
+
+#include "lgc/patch/Patch.h"
+#include "lgc/state/PipelineShaders.h"
+#include "llvm/IR/PassManager.h"
+
+namespace lgc {
+
+// =====================================================================================================================
+// Pass to generate tessellation control pass-through shader
+class TcsPassthroughShader : public llvm::PassInfoMixin<TcsPassthroughShader> {
+public:
+  llvm::PreservedAnalyses run(llvm::Module &module, llvm::ModuleAnalysisManager &analysisManager);
+
+  static llvm::StringRef name() { return "Patch LLVM for tessellation control pass-through shader generation"; }
+  void updatePipelineState(llvm::Module &module, PipelineState *pipelineState) const;
+  llvm::Function *generateTcsPassthroughShader(llvm::Module &module, PipelineShadersResult &pipelineShaders,
+                                               PipelineState *pipelineState);
+  llvm::Function *generateTcsPassthroughEntryPoint(llvm::Module &module, PipelineState *pipelineState);
+  void generateTcsPassthroughShaderBody(llvm::Module &module, PipelineShadersResult &pipelineShaders,
+                                        PipelineState *pipelineState, llvm::Function *entryPoint);
+};
+
+} // namespace lgc

lgc/include/lgc/state/AbiMetadata.h

@@ -594,6 +594,7 @@ enum class UserDataMapping : unsigned {
                                      //  pipeline stats query.
   StreamOutControlBuf = 0x10000016,  // 32-bit GPU virtual address to the streamout control buffer for GPUs that
                                      // use SW-emulated streamout.
+  SampleInfo = 0x10000018,           // Sample Info, numsamples + Sample Pattern
   ColorExportAddr = 0x10000020,      // Color export address
 
   // Values used in a user data PAL metadata register to be resolved at link time.

lgc/include/lgc/state/AbiUnlinked.h

@@ -77,14 +77,6 @@ const static char DescriptorTableOffset[] = "descset_";
 // The value of the relocation is whether resource is ResourceNodeType::DescriptorBufferCompact.
 const static char CompactBuffer[] = "compactbuffer_";
 
-// Number of samples is "$numsamples".
-// The value of the relocation is numSamples from the rasterizer state.
-const static char NumSamples[] = "$numsamples";
-
-// Sample pattern index is "$samplePatternIdx".
-// The value of the relocation is samplePatternIdx from the rasterizer state.
-const static char SamplePatternIdx[] = "$samplePatternIdx";
-
 // Device index is "$deviceIdx"
 // The value of the relocation is deviceIdx from the pipeline state.
 const static char DeviceIdx[] = "$deviceIdx";

lgc/include/lgc/state/Defs.h

@@ -72,6 +72,7 @@ const static char NggPrimShaderEntryPoint[] = "lgc.shader.PRIM.main";
 const static char EntryPointPrefix[] = "lgc.shader.";
 const static char CopyShaderEntryPoint[] = "lgc.shader.COPY.main";
 const static char NullFsEntryPoint[] = "lgc.shader.FS.null.main";
+const static char TcsPassthroughEntryPoint[] = "lgc.shader.TCS.passthrough.main";
 
 } // namespace lgcName
 

lgc/include/lgc/state/PipelineState.h

@@ -102,6 +102,7 @@ struct NggControl {
 // Represents transform feedback state metadata
 struct XfbStateMetadata {
   bool enableXfb;                                               // Whether transform feedback is active
+  bool enablePrimStats;                                         // Whether to count generated primitives
   std::array<unsigned, MaxTransformFeedbackBuffers> xfbStrides; // The strides of each XFB buffer
   std::array<int, MaxGsStreams> streamXfbBuffers;               // The stream-out XFB buffers bit mask per stream
   std::array<bool, MaxGsStreams> streamActive;                  // Flag indicating which vertex stream is active
@@ -188,6 +189,26 @@ class PipelineState final : public Pipeline {
   // Set the client-defined metadata to be stored inside the ELF
   void setClientMetadata(llvm::StringRef clientMetadata) override final;
 
+  // Set default tessellation inner/outer level from driver API
+  void setTessLevel(const float *tessLevelInner, const float *tessLevelOuter) override final {
+    m_tessLevel.inner[0] = tessLevelInner[0];
+    m_tessLevel.inner[1] = tessLevelInner[1];
+    m_tessLevel.outer[0] = tessLevelOuter[0];
+    m_tessLevel.outer[1] = tessLevelOuter[1];
+    m_tessLevel.outer[2] = tessLevelOuter[2];
+    m_tessLevel.outer[3] = tessLevelOuter[3];
+  }
+
+  // Get default tessellation inner/outer level from driver API
+  float getTessLevelInner(unsigned level) override final {
+    assert(level <= 2);
+    return m_tessLevel.inner[level];
+  }
+  float getTessLevelOuter(unsigned level) override final {
+    assert(level <= 4);
+    return m_tessLevel.outer[level];
+  }
+
   // -----------------------------------------------------------------------------------------------------------------
   // Other methods
 
@@ -375,6 +396,9 @@ class PipelineState final : public Pipeline {
   // Check if transform feedback is active
   bool enableXfb() const { return m_xfbStateMetadata.enableXfb; }
 
+  // Check if we need count primitives if XFB is disabled
+  bool enablePrimStats() const { return m_xfbStateMetadata.enablePrimStats; }
+
   // Get transform feedback strides
   const std::array<unsigned, MaxTransformFeedbackBuffers> &getXfbBufferStrides() const {
     return m_xfbStateMetadata.xfbStrides;
@@ -588,6 +612,12 @@ class PipelineState final : public Pipeline {
   bool m_outputPackState[ShaderStageGfxCount] = {}; // The output packable state per shader stage
   XfbStateMetadata m_xfbStateMetadata = {};         // Transform feedback state metadata
   llvm::SmallVector<unsigned, 32> m_userDataMaps[ShaderStageCountInternal]; // The user data per-shader
+  bool m_useMrt0AToMrtzA = false;                                           // Whether to copy mrt0.a to mrz.a
+
+  struct {
+    float inner[2]; // default tessellation inner level
+    float outer[4]; // default tessellation outer level
+  } m_tessLevel;
 };
 
 // =====================================================================================================================

lgc/include/lgc/state/ResourceUsage.h

@@ -594,8 +594,9 @@ struct InterfaceData {
 
       // Fragment shader
       struct {
-        unsigned viewIndex; // View Index
-        unsigned primMask;  // Primitive mask
+        unsigned viewIndex;  // View Index
+        unsigned primMask;   // Primitive mask
+        unsigned sampleInfo; // Sample Info: numSample + samplePattern
 
         // Perspective interpolation (I/J)
         struct {