Fix crash on recursive types. (shader-slang#5796)

Co-authored-by: Ellie Hermaszewska <[email protected]>

source/slang/slang-lower-to-ir.cpp

@@ -11384,6 +11384,12 @@ RefPtr<IRModule> generateIRForTranslationUnit(
 
     if (compileRequest->getLinkage()->m_optionSet.shouldRunNonEssentialValidation())
     {
+        // We don't allow recursive types.
+        checkForRecursiveTypes(module, compileRequest->getSink());
+
+        if (compileRequest->getSink()->getErrorCount() != 0)
+            return module;
+
         // Propagate `constexpr`-ness through the dataflow graph (and the
         // call graph) based on constraints imposed by different instructions.
         propagateConstExpr(module, compileRequest->getSink());
@@ -11395,10 +11401,6 @@ RefPtr<IRModule> generateIRForTranslationUnit(
         // instructions remain.
 
         checkForMissingReturns(module, compileRequest->getSink());
-
-        // We don't allow recursive types.
-        checkForRecursiveTypes(module, compileRequest->getSink());
-
         // Check for invalid differentiable function body.
         checkAutoDiffUsages(module, compileRequest->getSink());
 

tests/diagnostics/recursive-type.slang

@@ -0,0 +1,14 @@
+//DIAGNOSTIC_TEST:SIMPLE(filecheck=CHECK):-target spirv
+
+// CHECK: error 41001:
+
+struct Outer {
+    Outer next; // non-pointer
+    int y;
+};
+RWStructuredBuffer<Outer> Buf;
+
+[numthreads(1,1,1)]
+void csmain() {
+     Buf[0].y = 0;
+}