From 9936085aee3300072b4f0c17fa7cf41fe6d9238f Mon Sep 17 00:00:00 2001
From: Cameron Gutman <aicommander@gmail.com>
Date: Thu, 3 Oct 2024 20:28:51 -0500
Subject: [PATCH] Fix incorrect selection if pix_fmt is overridden before
 ffGetFormat()

The DRM renderer does this for the out-of-tree v4l2m2m patches to ensure
we get NV12 buffers, but it ends up clobbering our own preference for
DRM_PRIME frames in the process.
---
 app/streaming/video/ffmpeg.cpp | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/app/streaming/video/ffmpeg.cpp b/app/streaming/video/ffmpeg.cpp
index 5a4c6726..c17487e9 100644
--- a/app/streaming/video/ffmpeg.cpp
+++ b/app/streaming/video/ffmpeg.cpp
@@ -180,15 +180,25 @@ enum AVPixelFormat FFmpegVideoDecoder::ffGetFormat(AVCodecContext* context,
                                                    const enum AVPixelFormat* pixFmts)
 {
     FFmpegVideoDecoder* decoder = (FFmpegVideoDecoder*)context->opaque;
-    const enum AVPixelFormat *p;
+    const AVPixelFormat *p;
+    AVPixelFormat desiredFmt;
 
-    for (p = pixFmts; *p != -1; p++) {
+    if (decoder->m_HwDecodeCfg) {
+        desiredFmt = decoder->m_HwDecodeCfg->pix_fmt;
+    }
+    else if (decoder->m_RequiredPixelFormat != AV_PIX_FMT_NONE) {
+        desiredFmt = decoder->m_RequiredPixelFormat;
+    }
+    else {
+        desiredFmt = decoder->m_FrontendRenderer->getPreferredPixelFormat(decoder->m_VideoFormat);
+    }
+
+    for (p = pixFmts; *p != AV_PIX_FMT_NONE; p++) {
         // Only match our hardware decoding codec or preferred SW pixel
         // format (if not using hardware decoding). It's crucial
         // to override the default get_format() which will try
         // to gracefully fall back to software decode and break us.
-        if (*p == (decoder->m_HwDecodeCfg ? decoder->m_HwDecodeCfg->pix_fmt : context->pix_fmt) &&
-                decoder->m_BackendRenderer->prepareDecoderContextInGetFormat(context, *p)) {
+        if (*p == desiredFmt && decoder->m_BackendRenderer->prepareDecoderContextInGetFormat(context, *p)) {
             return *p;
         }
     }
@@ -196,7 +206,7 @@ enum AVPixelFormat FFmpegVideoDecoder::ffGetFormat(AVCodecContext* context,
     // Failed to match the preferred pixel formats. Try non-preferred pixel format options
     // for non-hwaccel decoders if we didn't have a required pixel format to use.
     if (decoder->m_HwDecodeCfg == nullptr && decoder->m_RequiredPixelFormat == AV_PIX_FMT_NONE) {
-        for (p = pixFmts; *p != -1; p++) {
+        for (p = pixFmts; *p != AV_PIX_FMT_NONE; p++) {
             if (decoder->m_FrontendRenderer->isPixelFormatSupported(decoder->m_VideoFormat, *p) &&
                     decoder->m_BackendRenderer->prepareDecoderContextInGetFormat(context, *p)) {
                 return *p;