Add metal downstream compiler + metallib target. (shader-slang#3990)

* Add metal downstream compiler + metallib target.

* Add more comments.

* Add missing override.

build/visual-studio/compiler-core/compiler-core.vcxproj

@@ -302,6 +302,7 @@
     <ClInclude Include="..\..\..\source\compiler-core\slang-lexer-diagnostic-defs.h" />
     <ClInclude Include="..\..\..\source\compiler-core\slang-lexer.h" />
     <ClInclude Include="..\..\..\source\compiler-core\slang-llvm-compiler.h" />
+    <ClInclude Include="..\..\..\source\compiler-core\slang-metal-compiler.h" />
     <ClInclude Include="..\..\..\source\compiler-core\slang-misc-diagnostic-defs.h" />
     <ClInclude Include="..\..\..\source\compiler-core\slang-name-convention-util.h" />
     <ClInclude Include="..\..\..\source\compiler-core\slang-name.h" />
@@ -352,6 +353,7 @@
     <ClCompile Include="..\..\..\source\compiler-core\slang-language-server-protocol.cpp" />
     <ClCompile Include="..\..\..\source\compiler-core\slang-lexer.cpp" />
     <ClCompile Include="..\..\..\source\compiler-core\slang-llvm-compiler.cpp" />
+    <ClCompile Include="..\..\..\source\compiler-core\slang-metal-compiler.cpp" />
     <ClCompile Include="..\..\..\source\compiler-core\slang-name-convention-util.cpp" />
     <ClCompile Include="..\..\..\source\compiler-core\slang-name.cpp" />
     <ClCompile Include="..\..\..\source\compiler-core\slang-nvrtc-compiler.cpp" />

build/visual-studio/compiler-core/compiler-core.vcxproj.filters

@@ -120,6 +120,9 @@
     <ClInclude Include="..\..\..\source\compiler-core\slang-llvm-compiler.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\compiler-core\slang-metal-compiler.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\compiler-core\slang-misc-diagnostic-defs.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -266,6 +269,9 @@
     <ClCompile Include="..\..\..\source\compiler-core\slang-llvm-compiler.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\source\compiler-core\slang-metal-compiler.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\source\compiler-core\slang-name-convention-util.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>

slang.h

@@ -610,6 +610,8 @@ extern "C"
         SLANG_HOST_HOST_CALLABLE,       ///< Host callable host code (ie non kernel/shader) 
         SLANG_CPP_PYTORCH_BINDING,      ///< C++ PyTorch binding code.
         SLANG_METAL,                    ///< Metal shading language
+        SLANG_METAL_LIB,                ///< Metal library
+        SLANG_METAL_LIB_ASM,            ///< Metal library assembly
         SLANG_TARGET_COUNT_OF,
     };
 

source/compiler-core/slang-artifact-desc-util.cpp

@@ -287,6 +287,8 @@ SLANG_HIERARCHICAL_ENUM(ArtifactStyle, SLANG_ARTIFACT_STYLE, SLANG_ARTIFACT_STYL
         case SLANG_OBJECT_CODE:             return Desc::make(Kind::ObjectCode, Payload::HostCPU, Style::Kernel, 0);
         case SLANG_HOST_HOST_CALLABLE:      return Desc::make(Kind::HostCallable, Payload::HostCPU, Style::Host, 0);
         case SLANG_METAL:                   return Desc::make(Kind::Source, Payload::Metal, Style::Kernel, 0);
+        case SLANG_METAL_LIB:               return Desc::make(Kind::Executable, Payload::MetalAIR, Style::Kernel, 0);
+        case SLANG_METAL_LIB_ASM:           return Desc::make(Kind::Assembly, Payload::MetalAIR, Style::Kernel, 0);
         default: break;
     }
 
@@ -328,6 +330,7 @@ SLANG_HIERARCHICAL_ENUM(ArtifactStyle, SLANG_ARTIFACT_STYLE, SLANG_ARTIFACT_STYL
                 case Payload::C:            return SLANG_C_SOURCE;
                 case Payload::Cpp:          return (desc.style == Style::Host) ? SLANG_HOST_CPP_SOURCE : SLANG_CPP_SOURCE;
                 case Payload::CUDA:         return SLANG_CUDA_SOURCE;
+                case Payload::Metal:        return SLANG_METAL;
                 default: break;
             }
             break;
@@ -340,6 +343,7 @@ SLANG_HIERARCHICAL_ENUM(ArtifactStyle, SLANG_ARTIFACT_STYLE, SLANG_ARTIFACT_STYL
                 case Payload::DXIL:         return SLANG_DXIL_ASM;
                 case Payload::DXBC:         return SLANG_DXBC_ASM;
                 case Payload::PTX:          return SLANG_PTX;
+                case Payload::MetalAIR:     return SLANG_METAL_LIB_ASM;
                 default: break;
             }
         }
@@ -367,6 +371,7 @@ SLANG_HIERARCHICAL_ENUM(ArtifactStyle, SLANG_ARTIFACT_STYLE, SLANG_ARTIFACT_STYL
                 case Payload::DXIL:         return SLANG_DXIL;
                 case Payload::DXBC:         return SLANG_DXBC;
                 case Payload::PTX:          return SLANG_PTX;
+                case Payload::MetalAIR:     return SLANG_METAL_LIB_ASM;
                 default: break;
             }
         }
@@ -638,7 +643,7 @@ static UnownedStringSlice _getPayloadExtension(ArtifactPayload payload)
 
         case Payload::SlangIR:      return toSlice("slang-ir");
 
-        case Payload::MetalAIR:     return toSlice("air");
+        case Payload::MetalAIR:     return toSlice("metallib");
 
         case Payload::PdbDebugInfo: return toSlice("pdb");
         case Payload::SourceMap:    return toSlice("map");

source/compiler-core/slang-downstream-compiler-util.cpp

@@ -23,6 +23,7 @@
 #include "slang-dxc-compiler.h"
 #include "slang-glslang-compiler.h"
 #include "slang-llvm-compiler.h"
+#include "slang-metal-compiler.h"
 
 namespace Slang
 {
@@ -330,6 +331,7 @@ DownstreamCompilerMatchVersion DownstreamCompilerUtil::getCompiledVersion()
     outFuncs[int(SLANG_PASS_THROUGH_SPIRV_OPT)] = &SpirvOptDownstreamCompilerUtil::locateCompilers;
     outFuncs[int(SLANG_PASS_THROUGH_LLVM)] = &LLVMDownstreamCompilerUtil::locateCompilers;
     outFuncs[int(SLANG_PASS_THROUGH_SPIRV_DIS)] = &SpirvDisDownstreamCompilerUtil::locateCompilers;
+    outFuncs[int(SLANG_PASS_THROUGH_METAL)] = &MetalDownstreamCompilerUtil::locateCompilers;
 }
 
 static String _getParentPath(const String& path)

source/compiler-core/slang-gcc-compiler-util.cpp

@@ -107,12 +107,15 @@ SlangResult GCCDownstreamCompilerUtil::calcVersion(const ExecutableLocation& exe
         UnownedStringSlice::fromLiteral("clang version"),
         UnownedStringSlice::fromLiteral("gcc version"),
         UnownedStringSlice::fromLiteral("Apple LLVM version"),
+        UnownedStringSlice::fromLiteral("Apple metal version"),
+
     };
     const SlangPassThrough types[] =
     {
         SLANG_PASS_THROUGH_CLANG,
         SLANG_PASS_THROUGH_GCC,
         SLANG_PASS_THROUGH_CLANG,
+        SLANG_PASS_THROUGH_METAL,
     };
 
     SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(prefixes) == SLANG_COUNT_OF(types));
@@ -262,8 +265,9 @@ static SlangResult _parseGCCFamilyLine(SliceAllocator& allocator, const UnownedS
         const auto split1 = split[1].trim();
         const auto text = split[2].trim();
 
-        // Check for special handling for clang (Can be Clang or clang apparently)
+        // Check for special handling for clang or metal
         if (split0.startsWith(UnownedStringSlice::fromLiteral("clang")) ||
+            split0.startsWith(UnownedStringSlice::fromLiteral("metal")) ||
             split0.startsWith(UnownedStringSlice::fromLiteral("Clang")) ||
             split0 == UnownedStringSlice::fromLiteral("g++") ||
             split0 == UnownedStringSlice::fromLiteral("gcc"))
@@ -469,6 +473,11 @@ static SlangResult _parseGCCFamilyLine(SliceAllocator& allocator, const UnownedS
         // C++17 since we share headers with slang itself (which uses c++17)
         cmdLine.addArg("-std=c++17");
     }
+
+    if (targetDesc.payload == ArtifactDesc::Payload::MetalAIR)
+    {
+        cmdLine.addArg("-std=macos-metal2.3");
+    }
 
     // Our generated code very often casts between dissimilar types with the
     // knowledge that they have the same representation. This is strictly

source/compiler-core/slang-metal-compiler.cpp

@@ -0,0 +1,80 @@
+#include "slang-metal-compiler.h"
+#include "slang-gcc-compiler-util.h"
+#include "slang-artifact-desc-util.h"
+#include "slang-artifact-util.h"
+#include "slang-artifact-representation.h"
+
+namespace Slang
+{
+
+    class MetalDownstreamCompiler : public DownstreamCompilerBase
+    {
+    public:
+        // Because the metal compiler shares the same commandline interface with clang,
+        // we will use GccDownstreamCompilerUtil, which implements both gcc and clang,
+        // to create the inner compiler and wrap it here.
+        //
+        ComPtr<IDownstreamCompiler> cppCompiler;
+        String executablePath;
+
+        MetalDownstreamCompiler(ComPtr<IDownstreamCompiler>& innerCompiler, String path)
+            : DownstreamCompilerBase(innerCompiler->getDesc())
+            , cppCompiler(innerCompiler)
+            , executablePath(path)
+        {
+        }
+
+        virtual SLANG_NO_THROW bool SLANG_MCALL isFileBased() override { return true; }
+
+        virtual SLANG_NO_THROW SlangResult SLANG_MCALL compile(const CompileOptions& options, IArtifact** outArtifact)
+        {
+            // All compile requests should be routed directly to the inner compiler.
+            return cppCompiler->compile(options, outArtifact);
+        }
+
+        virtual SLANG_NO_THROW bool SLANG_MCALL canConvert(const ArtifactDesc& from, const ArtifactDesc& to) override
+        {
+            // Report that we can convert Metal IR to disassembly.
+            return ArtifactDescUtil::isDisassembly(from, to) && from.payload == ArtifactPayload::MetalAIR;
+        }
+
+        virtual SLANG_NO_THROW SlangResult SLANG_MCALL convert(IArtifact* from, const ArtifactDesc& to, IArtifact** outArtifact) override
+        {
+            // Use metal-objdump to disassemble the Metal IR.
+
+            ExecutableLocation exeLocation(executablePath, "metal-objdump");
+            CommandLine cmdLine;
+            cmdLine.setExecutableLocation(exeLocation);
+            cmdLine.addArg("--disassemble");
+            ComPtr<IOSFileArtifactRepresentation> srcFile;
+            SLANG_RETURN_ON_FAIL(from->requireFile(IArtifact::Keep::No, srcFile.writeRef()));
+            cmdLine.addArg(String(srcFile->getPath()));
+
+            ExecuteResult exeRes;
+            SLANG_RETURN_ON_FAIL(ProcessUtil::execute(cmdLine, exeRes));
+            auto artifact = ArtifactUtil::createArtifact(to);
+            artifact->addRepresentationUnknown(StringBlob::create(exeRes.standardOutput));
+            *outArtifact = artifact.detach();
+            return SLANG_OK;
+        }
+    };
+
+    static SlangResult locateMetalCompiler(const String& path, DownstreamCompilerSet* set)
+    {
+        ComPtr<IDownstreamCompiler> innerCppCompiler;
+        ExecutableLocation exeLocation(path, "metal");
+        SLANG_RETURN_ON_FAIL(GCCDownstreamCompilerUtil::createCompiler(exeLocation, innerCppCompiler));
+
+        ComPtr<IDownstreamCompiler> compiler = ComPtr<IDownstreamCompiler>(
+            new MetalDownstreamCompiler(innerCppCompiler, path));
+        set->addCompiler(compiler);
+        return SLANG_OK;
+    }
+
+    SlangResult MetalDownstreamCompilerUtil::locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set)
+    {
+        SLANG_UNUSED(loader);
+        return locateMetalCompiler(path, set);
+    }
+
+}

source/compiler-core/slang-metal-compiler.h

@@ -0,0 +1,18 @@
+#ifndef SLANG_METAL_COMPILER_UTIL_H
+#define SLANG_METAL_COMPILER_UTIL_H
+
+#include "slang-downstream-compiler-util.h"
+
+#include "../core/slang-platform.h"
+
+namespace Slang
+{
+
+    struct MetalDownstreamCompilerUtil
+    {
+        static SlangResult locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set);
+    };
+
+}
+
+#endif

source/core/slang-type-text-util.cpp

@@ -61,7 +61,9 @@ static const TypeTextUtil::CompileTargetInfo s_compileTargetInfos[] =
     { SLANG_SHADER_HOST_CALLABLE,  "",                                          "host-callable,callable",   "Host callable" },
     { SLANG_OBJECT_CODE,    "obj,o",                                            "object-code",              "Object code" },
     { SLANG_HOST_HOST_CALLABLE, "",                                             "host-host-callable",       "Host callable for host execution" },
-    { SLANG_METAL,          "metal",                                            "metal",                    "Metal shader source"},
+    { SLANG_METAL,          "metal",                                            "metal",                    "Metal shader source" },
+    { SLANG_METAL_LIB,      "metallib",                                         "metallib",                 "Metal Library Bytecode" },
+    { SLANG_METAL_LIB_ASM,  "metallib-asm"                                      "metallib-asm",             "Metal Library Bytecode assembly" },
 };
 
 static const NamesDescriptionValue s_languageInfos[] =
@@ -88,6 +90,7 @@ static const NamesDescriptionValue s_compilerInfos[] =
     { SLANG_PASS_THROUGH_NVRTC,     "nvrtc",     "NVRTC CUDA compiler" },
     { SLANG_PASS_THROUGH_LLVM,      "llvm",     "LLVM/Clang `slang-llvm`" },
     { SLANG_PASS_THROUGH_SPIRV_OPT, "spirv-opt",  "spirv-tools SPIRV optimizer" },
+    { SLANG_PASS_THROUGH_METAL,     "metal",    "Metal shader compiler" },
 };
 
 static const NamesDescriptionValue s_archiveTypeInfos[] =

source/slang/slang-capabilities.capdef

@@ -95,6 +95,10 @@ def glsl_spirv_1_4 : glsl_spirv_1_3;
 def glsl_spirv_1_5 : glsl_spirv_1_4;
 def glsl_spirv_1_6 : glsl_spirv_1_5;
 
+def metallib_2_3 : metal;
+def metallib_2_4 : metallib_2_3;
+
+
 abstract stage;
 def vertex : stage;
 def fragment : stage;
@@ -572,6 +576,10 @@ alias DX_6_5 = sm_6_5;
 alias DX_6_6 = sm_6_6;
 alias DX_6_7 = sm_6_7;
 
+
+alias METAL_2_3 = metallib_2_3;
+alias METAL_2_4 = metallib_2_4;
+
 alias sm_2_0_GLSL_140 = sm_4_0 | glsl | spirv_1_0 | cuda | cpp;
 alias sm_2_0_GLSL_400 = sm_4_0 | glsl | spirv_1_0 | cuda | cpp;
 alias appendstructuredbuffer = sm_5_0 + raytracing_stages_compute_fragment;
@@ -679,4 +687,4 @@ alias all = _sm_6_7 + hlsl_nvapi
              + _GL_NV_ray_tracing_motion_blur + _GL_NV_shader_texture_footprint
           | spirv_1_5 + sm_6_7
              + ser + shaderclock + texturefootprint + fragmentshaderinterlock + spvGroupNonUniformPartitionedNV
-             + spvRayTracingMotionBlurNV + spvRayTracingMotionBlurNV;
+             + spvRayTracingMotionBlurNV + spvRayTracingMotionBlurNV;

source/slang/slang-compiler.cpp

@@ -530,6 +530,11 @@ namespace Slang
             {
                 return PassThroughMode::Glslang;
             }
+            case CodeGenTarget::MetalLib:
+            case CodeGenTarget::MetalLibAssembly:
+            {
+                return PassThroughMode::MetalC;
+            }
             case CodeGenTarget::ShaderHostCallable:
             case CodeGenTarget::ShaderSharedLibrary:
             case CodeGenTarget::HostExecutable:
@@ -957,6 +962,7 @@ namespace Slang
             case CodeGenTarget::DXBytecode:         return CodeGenTarget::HLSL;
             case CodeGenTarget::DXIL:               return CodeGenTarget::HLSL;
             case CodeGenTarget::SPIRV:              return CodeGenTarget::GLSL;
+            case CodeGenTarget::MetalLib:           return CodeGenTarget::Metal;
             default: break;
         }
         return CodeGenTarget::Unknown;
@@ -1546,6 +1552,7 @@ namespace Slang
             case CodeGenTarget::SPIRVAssembly:
             case CodeGenTarget::DXBytecodeAssembly:
             case CodeGenTarget::DXILAssembly:
+            case CodeGenTarget::MetalLibAssembly:
             {
                 // First compile to an intermediate target for the corresponding binary format.
                 const CodeGenTarget intermediateTarget = _getIntermediateTarget(target);
@@ -1573,6 +1580,7 @@ namespace Slang
                 [[fallthrough]];
             case CodeGenTarget::DXIL:
             case CodeGenTarget::DXBytecode:
+            case CodeGenTarget::MetalLib:
             case CodeGenTarget::PTX:
             case CodeGenTarget::ShaderHostCallable:
             case CodeGenTarget::ShaderSharedLibrary:
@@ -1602,6 +1610,8 @@ namespace Slang
         case CodeGenTarget::SPIRV:
         case CodeGenTarget::DXIL:
         case CodeGenTarget::DXBytecode:
+        case CodeGenTarget::MetalLib:
+        case CodeGenTarget::MetalLibAssembly:
         case CodeGenTarget::PTX:
         case CodeGenTarget::HostHostCallable:
         case CodeGenTarget::ShaderHostCallable:

source/slang/slang-compiler.h

@@ -93,6 +93,8 @@ namespace Slang
         ObjectCode          = SLANG_OBJECT_CODE,
         HostHostCallable    = SLANG_HOST_HOST_CALLABLE,
         Metal               = SLANG_METAL,
+        MetalLib            = SLANG_METAL_LIB,
+        MetalLibAssembly    = SLANG_METAL_LIB_ASM,
         CountOf             = SLANG_TARGET_COUNT_OF,
     };
 

source/slang/slang-emit-c-like.cpp

@@ -74,6 +74,8 @@ struct CLikeSourceEmitter::ComputeEmitActionsContext
         case CodeGenTarget::DXBytecodeAssembly:
         case CodeGenTarget::DXIL:
         case CodeGenTarget::DXILAssembly:
+        case CodeGenTarget::MetalLib:
+        case CodeGenTarget::MetalLibAssembly:
         {
             return SourceLanguage::Unknown;
         }

source/slang/slang-ir-link.cpp

@@ -1468,6 +1468,8 @@ static bool doesTargetAllowUnresolvedFuncSymbol(TargetRequest* req)
     {
     case CodeGenTarget::HLSL:
     case CodeGenTarget::Metal:
+    case CodeGenTarget::MetalLib:
+    case CodeGenTarget::MetalLibAssembly:
     case CodeGenTarget::DXIL:
     case CodeGenTarget::DXILAssembly:
     case CodeGenTarget::HostCPPSource:

source/slang/slang-options.cpp

@@ -2816,6 +2816,9 @@ SlangResult OptionsParser::_parse(
                     case CodeGenTarget::ShaderSharedLibrary:
                     case CodeGenTarget::PyTorchCppBinding:
                     case CodeGenTarget::DXIL:
+                    case CodeGenTarget::MetalLib:
+                    case CodeGenTarget::MetalLibAssembly:
+                    case CodeGenTarget::Metal:
                         rawOutput.isWholeProgram = true;
                         break;
                     case CodeGenTarget::SPIRV: