Commit 868922c2 for openh264

commit 868922c210d0d7b34a2a5dc2e89c71af28f352d3
Author: Andrew Osmond <aosmond@mozilla.com>
Date:   Sun Sep 20 03:50:08 2026 -0400

    Teach GMP encoder to notify on encoding failures. (#4017)

    EncodeFrame's return code was checked and logged, but eFrameType alone
    decided what happened next, so a hard failure was indistinguishable
    from OpenH264 skipping a frame for rate control. Gecko sees only a
    dropped frame, which it treats as legitimate, so a persistent failure
    looks like a stream that has quietly stopped producing output. This was
    found while debugging a web-platform test that timed out instead of
    reporting an error.

    Report GMPEncodeErr through the existing callback when EncodeFrame
    fails, or when it reports videoFrameTypeInvalid. This ends the encode
    session rather than dropping the frame.

diff --git a/module/gmp-openh264.cpp b/module/gmp-openh264.cpp
index e25750ba..db9b0577 100644
--- a/module/gmp-openh264.cpp
+++ b/module/gmp-openh264.cpp
@@ -603,6 +603,7 @@ class OpenH264VideoEncoder : public GMPVideoEncoder, public RefCounted {
   void Encode_w (GMPVideoi420Frame* inputImage,
                  GMPVideoFrameType frame_type) {
     SFrameBSInfo encoded;
+    memset (&encoded, 0, sizeof (encoded));

     if (frame_type  == kGMPKeyFrame) {
       encoder_->ForceIntraFrame (true);
@@ -633,8 +634,16 @@ class OpenH264VideoEncoder : public GMPVideoEncoder, public RefCounted {
     const SSourcePicture* pics = &src;

     int result = encoder_->EncodeFrame (pics, &encoded);
-    if (result != cmResultSuccess) {
-      GMPLOG (GL_ERROR, "Couldn't encode frame. Error = " << result);
+    if (result != cmResultSuccess || encoded.eFrameType == videoFrameTypeInvalid) {
+      GMPLOG (GL_ERROR, "Couldn't encode frame. Error = "
+              << result
+              << " Type = "
+              << encoded.eFrameType);
+      TrySyncRunOnMainThread (WrapTask (
+                                   this,
+                                   &OpenH264VideoEncoder::EncodeFailed_m,
+                                   inputImage));
+      return;
     }


@@ -660,11 +669,10 @@ class OpenH264VideoEncoder : public GMPVideoEncoder, public RefCounted {
       break;
     case videoFrameTypeIPMixed://this type is currently not suppported
     case videoFrameTypeInvalid:
-      GMPLOG (GL_ERROR, "Couldn't encode frame. Type = "
-              << encoded.eFrameType);
-      break;
     default:
       // The API is defined as returning a type.
+      GMPLOG (GL_ERROR, "Couldn't encode frame. Type = "
+              << encoded.eFrameType);
       assert (false);
       break;
     }
@@ -778,6 +786,12 @@ class OpenH264VideoEncoder : public GMPVideoEncoder, public RefCounted {
     frame->Destroy();
   }

+  // The frame must be destroyed, and the error reported, on the main thread.
+  void EncodeFailed_m (GMPVideoi420Frame* frame) {
+    frame->Destroy();
+    Error (GMPEncodeErr);
+  }
+

  private:
   GMPVideoHost* host_;