drm: Add workaround for NV12 scanout on SteamOS 3.6

src/Backends/DRMBackend.cpp

@@ -58,14 +58,28 @@ extern int g_nPreferredOutputWidth;
 extern int g_nPreferredOutputHeight;
 
 gamescope::ConVar<bool> cv_drm_single_plane_optimizations( "drm_single_plane_optimizations", true, "Whether or not to enable optimizations for single plane usage." );
+
 gamescope::ConVar<bool> cv_drm_debug_disable_shaper_and_3dlut( "drm_debug_disable_shaper_and_3dlut", false, "Shaper + 3DLUT chicken bit. (Force disable/DEFAULT, no logic change)" );
 gamescope::ConVar<bool> cv_drm_debug_disable_degamma_tf( "drm_debug_disable_degamma_tf", false, "Degamma chicken bit. (Forces DEGAMMA_TF to DEFAULT, does not affect other logic)" );
 gamescope::ConVar<bool> cv_drm_debug_disable_regamma_tf( "drm_debug_disable_regamma_tf", false, "Regamma chicken bit. (Forces REGAMMA_TF to DEFAULT, does not affect other logic)" );
 gamescope::ConVar<bool> cv_drm_debug_disable_output_tf( "drm_debug_disable_output_tf", false, "Force default (identity) output TF, affects other logic. Not a property directly." );
 gamescope::ConVar<bool> cv_drm_debug_disable_blend_tf( "drm_debug_disable_blend_tf", false, "Blending chicken bit. (Forces BLEND_TF to DEFAULT, does not affect other logic)" );
+gamescope::ConVar<bool> cv_drm_debug_disable_ctm( "drm_debug_disable_ctm", false, "CTM chicken bit. (Forces CTM off, does not affect other logic)" );
+gamescope::ConVar<bool> cv_drm_debug_disable_color_encoding( "drm_debug_disable_color_encoding", false, "YUV Color Encoding chicken bit. (Forces COLOR_ENCODING to DEFAULT, does not affect other logic)" );
+gamescope::ConVar<bool> cv_drm_debug_disable_color_range( "drm_debug_disable_color_range", false, "YUV Color Range chicken bit. (Forces COLOR_RANGE to DEFAULT, does not affect other logic)" );
 gamescope::ConVar<bool> cv_drm_debug_disable_explicit_sync( "drm_debug_disable_explicit_sync", false, "Force disable explicit sync on the DRM backend." );
 gamescope::ConVar<bool> cv_drm_debug_disable_in_fence_fd( "drm_debug_disable_in_fence_fd", false, "Force disable IN_FENCE_FD being set to avoid over-synchronization on the DRM backend." );
 
+// HACK:
+// Workaround for AMDGPU bug on SteamOS 3.6 right now.
+// Using a Shaper or 3D LUT results in the commit failing, and we really want
+// NV12 direct scanout so we can get GFXOFF.
+// The compromise here is that colors may look diff to when we composite due to
+// lack of 3D LUT, etc.
+// TODO: Come back to me on the kernel side after figuring out what broke
+// since we moved to the upstream properites and a bunch of work happened.
+gamescope::ConVar<bool> cv_drm_hack_nv12_color_mgmt_fix( "drm_hack_nv12_color_mgmt_fix", true, "If using NV12, disable explicit degamma + shaper + 3D LUT" );
+
 namespace gamescope
 {
 	std::tuple<int32_t, int32_t, int32_t> GetKernelVersion()
@@ -2388,14 +2402,23 @@ drm_prepare_liftoff( struct drm_t *drm, const struct FrameInfo_t *frameInfo, boo
 
 			if ( frameInfo->layers[i].applyColorMgmt )
 			{
-				if ( entry.layerState[i].ycbcr )
+				bool bYCbCr = entry.layerState[i].ycbcr;
+
+				if ( !cv_drm_debug_disable_color_encoding && bYCbCr )
 				{
 					liftoff_layer_set_property( drm->lo_layers[ i ], "COLOR_ENCODING", entry.layerState[i].colorEncoding );
-					liftoff_layer_set_property( drm->lo_layers[ i ], "COLOR_RANGE",    entry.layerState[i].colorRange );
 				}
 				else
 				{
 					liftoff_layer_unset_property( drm->lo_layers[ i ], "COLOR_ENCODING" );
+				}
+
+				if ( !cv_drm_debug_disable_color_range && bYCbCr )
+				{
+					liftoff_layer_set_property( drm->lo_layers[ i ], "COLOR_RANGE",    entry.layerState[i].colorRange );
+				}
+				else
+				{
 					liftoff_layer_unset_property( drm->lo_layers[ i ], "COLOR_RANGE" );
 				}
 
@@ -2404,7 +2427,7 @@ drm_prepare_liftoff( struct drm_t *drm, const struct FrameInfo_t *frameInfo, boo
 					amdgpu_transfer_function degamma_tf = colorspace_to_plane_degamma_tf( entry.layerState[i].colorspace );
 					amdgpu_transfer_function shaper_tf = colorspace_to_plane_shaper_tf( entry.layerState[i].colorspace );
 
-					if ( entry.layerState[i].ycbcr )
+					if ( bYCbCr )
 					{
 						// JoshA: Based on the Steam In-Home Streaming Shader,
 						// it looks like Y is actually sRGB, not HDTV G2.4
@@ -2418,12 +2441,20 @@ drm_prepare_liftoff( struct drm_t *drm, const struct FrameInfo_t *frameInfo, boo
 						shaper_tf = AMDGPU_TRANSFER_FUNCTION_BT709_INV_OETF;
 					}
 
-					if (!cv_drm_debug_disable_degamma_tf)
+					bool bUseDegamma = !cv_drm_debug_disable_degamma_tf;
+					if ( bYCbCr && cv_drm_hack_nv12_color_mgmt_fix )
+						bUseDegamma = false;
+
+					if ( bUseDegamma )
 						liftoff_layer_set_property( drm->lo_layers[ i ], "AMD_PLANE_DEGAMMA_TF", degamma_tf );
 					else
 						liftoff_layer_set_property( drm->lo_layers[ i ], "AMD_PLANE_DEGAMMA_TF", 0 );
 
-					if ( !cv_drm_debug_disable_shaper_and_3dlut )
+					bool bUseShaperAnd3DLUT = !cv_drm_debug_disable_shaper_and_3dlut;
+					if ( bYCbCr && cv_drm_hack_nv12_color_mgmt_fix )
+						bUseShaperAnd3DLUT = false;
+
+					if ( bUseShaperAnd3DLUT )
 					{
 						liftoff_layer_set_property( drm->lo_layers[ i ], "AMD_PLANE_SHAPER_LUT", drm->pending.shaperlut_id[ ColorSpaceToEOTFIndex( entry.layerState[i].colorspace ) ]->GetBlobValue() );
 						liftoff_layer_set_property( drm->lo_layers[ i ], "AMD_PLANE_SHAPER_TF", shaper_tf );
@@ -2457,7 +2488,7 @@ drm_prepare_liftoff( struct drm_t *drm, const struct FrameInfo_t *frameInfo, boo
 				else
 					liftoff_layer_set_property( drm->lo_layers[ i ], "AMD_PLANE_BLEND_TF", AMDGPU_TRANSFER_FUNCTION_DEFAULT );
 
-				if (frameInfo->layers[i].ctm != nullptr)
+				if (!cv_drm_debug_disable_ctm && frameInfo->layers[i].ctm != nullptr)
 					liftoff_layer_set_property( drm->lo_layers[ i ], "AMD_PLANE_CTM", frameInfo->layers[i].ctm->GetBlobValue() );
 				else
 					liftoff_layer_set_property( drm->lo_layers[ i ], "AMD_PLANE_CTM", 0 );