vk: fix crash when enabling/disabling EmulateFramebuffer

Make sure to end the current render pass with the previous setting. Fix initial layout of color attachments in OIT. Fix missing initial transition after Term/Init in !OIT. Issue #1734
flyinghead · Nov 11, 2024 · 3efb87b · 3efb87b
1 parent c9f2383
commit 3efb87b
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 24 deletions.
diff --git a/core/rend/vulkan/drawer.cpp b/core/rend/vulkan/drawer.cpp
@@ -636,6 +636,7 @@ void TextureDrawer::EndRenderPass()
 
 void ScreenDrawer::Init(SamplerManager *samplerManager, ShaderManager *shaderManager, const vk::Extent2D& viewport)
 {
+	emulateFramebuffer = config::EmulateFramebuffer;
 	this->shaderManager = shaderManager;
 	if (this->viewport != viewport)
 	{
@@ -747,7 +748,7 @@ vk::CommandBuffer ScreenDrawer::BeginRenderPass()
 		{
 			setImageLayout(commandBuffer, colorAttachments[GetCurrentImage()]->GetImage(), vk::Format::eR8G8B8A8Unorm,
 					1, vk::ImageLayout::eUndefined,
-					config::EmulateFramebuffer ? vk::ImageLayout::eTransferSrcOptimal : vk::ImageLayout::eShaderReadOnlyOptimal);
+					emulateFramebuffer ? vk::ImageLayout::eTransferSrcOptimal : vk::ImageLayout::eShaderReadOnlyOptimal);
 			transitionNeeded[GetCurrentImage()] = false;
 		}
 
@@ -774,7 +775,7 @@ void ScreenDrawer::EndRenderPass()
 	if (!renderPassStarted)
 		return;
 	currentCommandBuffer.endRenderPass();
-	if (config::EmulateFramebuffer)
+	if (emulateFramebuffer)
 	{
 		scaleAndWriteFramebuffer(currentCommandBuffer, colorAttachments[GetCurrentImage()].get());
 	}

diff --git a/core/rend/vulkan/drawer.h b/core/rend/vulkan/drawer.h
@@ -298,6 +298,8 @@ class ScreenDrawer : public Drawer
 		framebuffers.clear();
 		colorAttachments.clear();
 		depthAttachment.reset();
+		transitionNeeded.clear();
+		clearNeeded.clear();
 		Drawer::Term();
 	}
 
@@ -333,6 +335,7 @@ class ScreenDrawer : public Drawer
 	std::vector<bool> clearNeeded;
 	bool frameRendered = false;
 	float aspectRatio = 0.f;
+	bool emulateFramebuffer = false;
 };
 
 class TextureDrawer : public Drawer

diff --git a/core/rend/vulkan/oit/oit_drawer.cpp b/core/rend/vulkan/oit/oit_drawer.cpp
@@ -296,9 +296,8 @@ bool OITDrawer::Draw(const Texture *fogTexture, const Texture *paletteTexture)
 		needAttachmentTransition = false;
 		// Not convinced that this is really needed but it makes validation layers happy
 		for (auto& attachment : colorAttachments)
-			// FIXME should be eTransferSrcOptimal if fullFB (screen) or copy to vram (rtt) -> 1 validation error at startup
 			setImageLayout(cmdBuffer, attachment->GetImage(), vk::Format::eR8G8B8A8Unorm, 1,
-					vk::ImageLayout::eUndefined, vk::ImageLayout::eShaderReadOnlyOptimal);
+					vk::ImageLayout::eUndefined, getAttachmentInitialLayout());
 		for (auto& attachment : depthAttachments)
 			setImageLayout(cmdBuffer, attachment->GetImage(), GetContext()->GetDepthFormat(), 1,
 					vk::ImageLayout::eUndefined, vk::ImageLayout::eDepthStencilReadOnlyOptimal);
@@ -556,7 +555,7 @@ void OITScreenDrawer::MakeFramebuffers(const vk::Extent2D& viewport)
 	// make sure all attachments have the same dimensions
 	maxWidth = 0;
 	maxHeight = 0;
-	MakeBuffers(viewport.width, viewport.height, config::EmulateFramebuffer ? vk::ImageUsageFlagBits::eTransferSrc : vk::ImageUsageFlagBits::eSampled);
+	MakeBuffers(viewport.width, viewport.height, emulateFramebuffer ? vk::ImageUsageFlagBits::eTransferSrc : vk::ImageUsageFlagBits::eSampled);
 
 	clearNeeded = { true, true };
 }

diff --git a/core/rend/vulkan/oit/oit_drawer.h b/core/rend/vulkan/oit/oit_drawer.h
@@ -83,6 +83,7 @@ class OITDrawer : public BaseDrawer
 	void MakeBuffers(int width, int height, vk::ImageUsageFlags colorUsage = {});
 	virtual vk::Framebuffer getFramebuffer(int renderPass, int renderPassCount) = 0;
 	int getFramebufferIndex() { return framebufferIndex; }
+	virtual vk::ImageLayout getAttachmentInitialLayout() = 0;
 
 	vk::Rect2D viewport;
 	std::array<std::unique_ptr<FramebufferAttachment>, 2> colorAttachments;
@@ -138,6 +139,7 @@ class OITScreenDrawer : public OITDrawer
 	void Init(SamplerManager *samplerManager, OITShaderManager *shaderManager, OITBuffers *oitBuffers,
 			const vk::Extent2D& viewport)
 	{
+		emulateFramebuffer = config::EmulateFramebuffer;
 		if (!screenPipelineManager) {
 			screenPipelineManager = std::make_unique<OITPipelineManager>();
 			screenPipelineManager->Init(shaderManager, oitBuffers);
@@ -160,7 +162,7 @@ class OITScreenDrawer : public OITDrawer
 		if (!frameStarted)
 			return;
 		frameStarted = false;
-		if (config::EmulateFramebuffer) {
+		if (emulateFramebuffer) {
 			scaleAndWriteFramebuffer(currentCommandBuffer, colorAttachments[framebufferIndex].get());
 		}
 		else
@@ -189,6 +191,9 @@ class OITScreenDrawer : public OITDrawer
 
 protected:
 	vk::Framebuffer getFramebuffer(int renderPass, int renderPassCount) override;
+	vk::ImageLayout getAttachmentInitialLayout() override {
+		return emulateFramebuffer ? vk::ImageLayout::eTransferSrcOptimal : vk::ImageLayout::eShaderReadOnlyOptimal;
+	}
 
 private:
 	void MakeFramebuffers(const vk::Extent2D& viewport);
@@ -197,6 +202,7 @@ class OITScreenDrawer : public OITDrawer
 	bool frameRendered = false;
 	float aspectRatio = 0.f;
 	bool frameStarted = false;
+	bool emulateFramebuffer = false;
 };
 
 class OITTextureDrawer : public OITDrawer
@@ -225,6 +231,9 @@ class OITTextureDrawer : public OITDrawer
 protected:
 	vk::CommandBuffer NewFrame() override;
 	vk::Framebuffer getFramebuffer(int renderPass, int renderPassCount) override;
+	vk::ImageLayout getAttachmentInitialLayout() override {
+		return config::RenderToTextureBuffer ? vk::ImageLayout::eTransferSrcOptimal : vk::ImageLayout::eShaderReadOnlyOptimal;
+	}
 
 private:
 	u32 textureAddr = 0;

diff --git a/core/rend/vulkan/oit/oit_renderer.cpp b/core/rend/vulkan/oit/oit_renderer.cpp
@@ -64,22 +64,24 @@ class OITVulkanRenderer final : public BaseVulkanRenderer
 
 	void Process(TA_context* ctx) override
 	{
-		if (ctx->rend.isRTT)
+		if (emulateFramebuffer != config::EmulateFramebuffer)
+		{
+			screenDrawer.EndFrame();
+			VulkanContext::Instance()->WaitIdle();
+			screenDrawer.Term();
+			screenDrawer.Init(&samplerManager, &oitShaderManager, &oitBuffers, viewport);
+			BaseInit(screenDrawer.GetRenderPass(), 2);
+			emulateFramebuffer = config::EmulateFramebuffer;
+		}
+		else if (ctx->rend.isRTT) {
 			screenDrawer.EndFrame();
+		}
 		BaseVulkanRenderer::Process(ctx);
 	}
 
 	bool Render() override
 	{
 		try {
-			if (emulateFramebuffer != config::EmulateFramebuffer)
-			{
-				VulkanContext::Instance()->WaitIdle();
-				screenDrawer.Term();
-				screenDrawer.Init(&samplerManager, &oitShaderManager, &oitBuffers, viewport);
-				BaseInit(screenDrawer.GetRenderPass(), 2);
-				emulateFramebuffer = config::EmulateFramebuffer;
-			}
 			OITDrawer *drawer;
 			if (pvrrc.isRTT)
 				drawer = &textureDrawer;

diff --git a/core/rend/vulkan/vulkan_renderer.cpp b/core/rend/vulkan/vulkan_renderer.cpp
@@ -338,22 +338,24 @@ class VulkanRenderer final : public BaseVulkanRenderer
 
 	void Process(TA_context* ctx) override
 	{
-		if (ctx->rend.isRTT)
+		if (emulateFramebuffer != config::EmulateFramebuffer)
+		{
+			screenDrawer.EndRenderPass();
+			VulkanContext::Instance()->WaitIdle();
+			screenDrawer.Term();
+			screenDrawer.Init(&samplerManager, &shaderManager, viewport);
+			BaseInit(screenDrawer.GetRenderPass());
+			emulateFramebuffer = config::EmulateFramebuffer;
+		}
+		else if (ctx->rend.isRTT) {
 			screenDrawer.EndRenderPass();
+		}
 		BaseVulkanRenderer::Process(ctx);
 	}
 
 	bool Render() override
 	{
 		try {
-			if (emulateFramebuffer != config::EmulateFramebuffer)
-			{
-				VulkanContext::Instance()->WaitIdle();
-				screenDrawer.Term();
-				screenDrawer.Init(&samplerManager, &shaderManager, viewport);
-				BaseInit(screenDrawer.GetRenderPass());
-				emulateFramebuffer = config::EmulateFramebuffer;
-			}
 			Drawer *drawer;
 			if (pvrrc.isRTT)
 				drawer = &textureDrawer;