Issue #2357 - VPXDecoder does not decode alpha frames.

Another requirement to have transparency in videos. Straight port of Firefox 53 implementation, save for some oddities relating to needing to put uint8_t instead of uint8 in yuv_convert to get this to compile.

Ref: BZ 1321076, 1329104

9 files changed, 315 insertions, 65 deletions

diff --git a/dom/media/MediaData.cpp b/dom/media/MediaData.cpp
index c113b62fe6..9f1205a0e1 100644
--- a/dom/media/MediaData.cpp
+++ b/dom/media/MediaData.cpp
@@ -7,6 +7,8 @@
 #include "MediaInfo.h"
 #include "VideoUtils.h"
 #include "ImageContainer.h"
+#include "mozilla/layers/SharedRGBImage.h"
+#include "YCbCrUtils.h"
 
 #include <stdint.h>
 
@@ -89,6 +91,45 @@ ValidatePlane(const VideoData::YCbCrBuffer::Plane& aPlane)
          aPlane.mStride > 0 && aPlane.mWidth <= aPlane.mStride;
 }
 
+static bool ValidateBufferAndPicture(const VideoData::YCbCrBuffer& aBuffer,
+                                     const IntRect& aPicture)
+{
+  // The following situation should never happen unless there is a bug
+  // in the decoder
+  if (aBuffer.mPlanes[1].mWidth != aBuffer.mPlanes[2].mWidth ||
+      aBuffer.mPlanes[1].mHeight != aBuffer.mPlanes[2].mHeight) {
+    NS_ERROR("C planes with different sizes");
+    return false;
+  }
+
+  // The following situations could be triggered by invalid input
+  if (aPicture.width <= 0 || aPicture.height <= 0) {
+    // In debug mode, makes the error more noticeable
+    MOZ_ASSERT(false, "Empty picture rect");
+    return false;
+  }
+  if (!ValidatePlane(aBuffer.mPlanes[0]) ||
+      !ValidatePlane(aBuffer.mPlanes[1]) ||
+      !ValidatePlane(aBuffer.mPlanes[2])) {
+    NS_WARNING("Invalid plane size");
+    return false;
+  }
+
+  // Ensure the picture size specified in the headers can be extracted out of
+  // the frame we've been supplied without indexing out of bounds.
+  CheckedUint32 xLimit = aPicture.x + CheckedUint32(aPicture.width);
+  CheckedUint32 yLimit = aPicture.y + CheckedUint32(aPicture.height);
+  if (!xLimit.isValid() || xLimit.value() > aBuffer.mPlanes[0].mStride ||
+      !yLimit.isValid() || yLimit.value() > aBuffer.mPlanes[0].mHeight)
+  {
+    // The specified picture dimensions can't be contained inside the video
+    // frame, we'll stomp memory if we try to copy it. Fail.
+    NS_WARNING("Overflowing picture rect");
+    return false;
+  }
+  return true;
+}
+
 VideoData::VideoData(int64_t aOffset,
                      int64_t aTime,
                      int64_t aDuration,
@@ -242,36 +283,7 @@ VideoData::CreateAndCopyData(const VideoInfo& aInfo,
     return v.forget();
   }
 
-  // The following situation should never happen unless there is a bug
-  // in the decoder
-  if (aBuffer.mPlanes[1].mWidth != aBuffer.mPlanes[2].mWidth ||
-      aBuffer.mPlanes[1].mHeight != aBuffer.mPlanes[2].mHeight) {
-    NS_ERROR("C planes with different sizes");
-    return nullptr;
-  }
-
-  // The following situations could be triggered by invalid input
-  if (aPicture.width <= 0 || aPicture.height <= 0) {
-    // In debug mode, makes the error more noticeable
-    MOZ_ASSERT(false, "Empty picture rect");
-    return nullptr;
-  }
-  if (!ValidatePlane(aBuffer.mPlanes[0]) || !ValidatePlane(aBuffer.mPlanes[1]) ||
-      !ValidatePlane(aBuffer.mPlanes[2])) {
-    NS_WARNING("Invalid plane size");
-    return nullptr;
-  }
-
-  // Ensure the picture size specified in the headers can be extracted out of
-  // the frame we've been supplied without indexing out of bounds.
-  CheckedUint32 xLimit = aPicture.x + CheckedUint32(aPicture.width);
-  CheckedUint32 yLimit = aPicture.y + CheckedUint32(aPicture.height);
-  if (!xLimit.isValid() || xLimit.value() > aBuffer.mPlanes[0].mStride ||
-      !yLimit.isValid() || yLimit.value() > aBuffer.mPlanes[0].mHeight)
-  {
-    // The specified picture dimensions can't be contained inside the video
-    // frame, we'll stomp memory if we try to copy it. Fail.
-    NS_WARNING("Overflowing picture rect");
+  if (!ValidateBufferAndPicture(aBuffer, aPicture)) {
     return nullptr;
   }
 
@@ -307,6 +319,73 @@ VideoData::CreateAndCopyData(const VideoInfo& aInfo,
 
 /* static */
 already_AddRefed<VideoData>
+VideoData::CreateAndCopyData(const VideoInfo& aInfo,
+                             ImageContainer* aContainer,
+                             int64_t aOffset,
+                             int64_t aTime,
+                             int64_t aDuration,
+                             const YCbCrBuffer& aBuffer,
+                             const YCbCrBuffer::Plane &aAlphaPlane,
+                             bool aKeyframe,
+                             int64_t aTimecode,
+                             const IntRect& aPicture)
+{
+  if (!aContainer) {
+    // Create a dummy VideoData with no image. This gives us something to
+    // send to media streams if necessary.
+    RefPtr<VideoData> v(new VideoData(aOffset,
+                                      aTime,
+                                      aDuration,
+                                      aKeyframe,
+                                      aTimecode,
+                                      aInfo.mDisplay,
+                                      0));
+    return v.forget();
+  }
+
+  if (!ValidateBufferAndPicture(aBuffer, aPicture)) {
+    return nullptr;
+  }
+
+  RefPtr<VideoData> v(new VideoData(aOffset,
+                                    aTime,
+                                    aDuration,
+                                    aKeyframe,
+                                    aTimecode,
+                                    aInfo.mDisplay,
+                                    0));
+
+  // Convert from YUVA to BGRA format on the software side.
+  RefPtr<layers::SharedRGBImage> videoImage =
+    aContainer->CreateSharedRGBImage();
+  v->mImage = videoImage;
+
+  if (!v->mImage) {
+    return nullptr;
+  }
+  if (!videoImage->Allocate(IntSize(aBuffer.mPlanes[0].mWidth,
+                                    aBuffer.mPlanes[0].mHeight),
+                            SurfaceFormat::B8G8R8A8)) {
+    return nullptr;
+  }
+  uint8_t* argb_buffer = videoImage->GetBuffer();
+  IntSize size = videoImage->GetSize();
+
+  // The naming convention for libyuv and associated utils is word-order.
+  // The naming convention in the gfx stack is byte-order.
+  ConvertYCbCrAToARGB(aBuffer.mPlanes[0].mData,
+                      aBuffer.mPlanes[1].mData,
+                      aBuffer.mPlanes[2].mData,
+                      aAlphaPlane.mData,
+                      aBuffer.mPlanes[0].mStride, aBuffer.mPlanes[1].mStride,
+                      argb_buffer, size.width * 4,
+                      size.width, size.height);
+
+  return v.forget();
+}
+
+/* static */
+already_AddRefed<VideoData>
 VideoData::CreateFromImage(const VideoInfo& aInfo,
                            int64_t aOffset,
                            int64_t aTime,
diff --git a/dom/media/MediaData.h b/dom/media/MediaData.h
index 7e93541baa..02a7162f42 100644
--- a/dom/media/MediaData.h
+++ b/dom/media/MediaData.h
@@ -474,6 +474,17 @@ public:
                                                        int64_t aTimecode,
                                                        const IntRect& aPicture);
 
+  static already_AddRefed<VideoData> CreateAndCopyData(const VideoInfo& aInfo,
+                                                       ImageContainer* aContainer,
+                                                       int64_t aOffset,
+                                                       int64_t aTime,
+                                                       int64_t aDuration,
+                                                       const YCbCrBuffer &aBuffer,
+                                                       const YCbCrBuffer::Plane &aAlphaPlane,
+                                                       bool aKeyframe,
+                                                       int64_t aTimecode,
+                                                       const IntRect& aPicture);
+
   static already_AddRefed<VideoData> CreateAndCopyIntoTextureClient(const VideoInfo& aInfo,
                                                                     int64_t aOffset,
                                                                     int64_t aTime,
diff --git a/dom/media/platforms/agnostic/VPXDecoder.cpp b/dom/media/platforms/agnostic/VPXDecoder.cpp
index 007ead0c7d..00cf7d85cc 100644
--- a/dom/media/platforms/agnostic/VPXDecoder.cpp
+++ b/dom/media/platforms/agnostic/VPXDecoder.cpp
@@ -35,6 +35,38 @@ static VPXDecoder::Codec MimeTypeToCodec(const nsACString& aMimeType)
   return VPXDecoder::Codec::Unknown;
 }
 
+static nsresult
+InitContext(vpx_codec_ctx_t* aCtx,
+            const VideoInfo& aInfo,
+            const VPXDecoder::Codec aCodec)
+{
+  int decode_threads = 2;
+
+  vpx_codec_iface_t* dx = nullptr;
+  if (aCodec == VPXDecoder::Codec::VP8) {
+    dx = vpx_codec_vp8_dx();
+  }
+  else if (aCodec == VPXDecoder::Codec::VP9) {
+    dx = vpx_codec_vp9_dx();
+    if (aInfo.mDisplay.width >= 2048) {
+      decode_threads = 8;
+    }
+    else if (aInfo.mDisplay.width >= 1024) {
+      decode_threads = 4;
+    }
+  }
+  decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors());
+
+  vpx_codec_dec_cfg_t config;
+  config.threads = decode_threads;
+  config.w = config.h = 0; // set after decode
+
+  if (!dx || vpx_codec_dec_init(aCtx, dx, &config, 0)) {
+    return NS_ERROR_FAILURE;
+  }
+  return NS_OK;
+}
+
 VPXDecoder::VPXDecoder(const CreateDecoderParams& aParams)
   : mImageContainer(aParams.mImageContainer)
   , mTaskQueue(aParams.mTaskQueue)
@@ -45,6 +77,7 @@ VPXDecoder::VPXDecoder(const CreateDecoderParams& aParams)
 {
   MOZ_COUNT_CTOR(VPXDecoder);
   PodZero(&mVPX);
+  PodZero(&mVPXAlpha);
 }
 
 VPXDecoder::~VPXDecoder()
@@ -56,34 +89,24 @@ void
 VPXDecoder::Shutdown()
 {
   vpx_codec_destroy(&mVPX);
+  vpx_codec_destroy(&mVPXAlpha);
 }
 
 RefPtr<MediaDataDecoder::InitPromise>
 VPXDecoder::Init()
 {
-  int decode_threads = 2;
-
-  vpx_codec_iface_t* dx = nullptr;
-  if (mCodec == Codec::VP8) {
-    dx = vpx_codec_vp8_dx();
-  } else if (mCodec == Codec::VP9) {
-    dx = vpx_codec_vp9_dx();
-    if (mInfo.mDisplay.width >= 2048) {
-      decode_threads = 8;
-    } else if (mInfo.mDisplay.width >= 1024) {
-      decode_threads = 4;
-    }
+  if (NS_FAILED(InitContext(&mVPX, mInfo, mCodec))) {
+    return VPXDecoder::InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+                                                    __func__);
   }
-  decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors());
-
-  vpx_codec_dec_cfg_t config;
-  config.threads = decode_threads;
-  config.w = config.h = 0; // set after decode
-
-  if (!dx || vpx_codec_dec_init(&mVPX, dx, &config, 0)) {
-    return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+  if (mInfo.HasAlpha()) {
+    if (NS_FAILED(InitContext(&mVPXAlpha, mInfo, mCodec))) {
+      return VPXDecoder::InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+                                                      __func__);
+    }
   }
-  return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
+  return VPXDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack,
+                                                   __func__);
 }
 
 void
@@ -115,14 +138,27 @@ VPXDecoder::DoDecode(MediaRawData* aSample)
       RESULT_DETAIL("VPX error: %s", vpx_codec_err_to_string(r)));
   }
 
-  vpx_codec_iter_t  iter = nullptr;
-  vpx_image_t      *img;
+  vpx_codec_iter_t iter = nullptr;
+  vpx_image_t *img;
+  vpx_image_t *img_alpha = nullptr;
+  bool alpha_decoded = false;
 
   while ((img = vpx_codec_get_frame(&mVPX, &iter))) {
     NS_ASSERTION(img->fmt == VPX_IMG_FMT_I420 ||
                  img->fmt == VPX_IMG_FMT_I444,
                  "WebM image format not I420 or I444");
-
+    NS_ASSERTION(!alpha_decoded,
+                 "Multiple frames per packet that contains alpha");
+ 
+    if (aSample->AlphaSize() > 0) {
+      if(!alpha_decoded){
+        MediaResult rv = DecodeAlpha(&img_alpha, aSample);
+        if (NS_FAILED(rv)) {
+          return(rv);
+        }
+        alpha_decoded = true;
+      }
+    }
     // Chroma shifts are rounded down as per the decoding examples in the SDK
     VideoData::YCbCrBuffer b;
     b.mPlanes[0].mData = img->planes[0];
@@ -174,17 +210,38 @@ VPXDecoder::DoDecode(MediaRawData* aSample)
     }();
     // TODO: need a newer libvpx to support full color range
 
-    RefPtr<VideoData> v =
-      VideoData::CreateAndCopyData(mInfo,
-                                   mImageContainer,
-                                   aSample->mOffset,
-                                   aSample->mTime,
-                                   aSample->mDuration,
-                                   b,
-                                   aSample->mKeyframe,
-                                   aSample->mTimecode,
-                                   mInfo.ScaledImageRect(img->d_w,
-                                                         img->d_h));
+    RefPtr<VideoData> v;
+    if (!img_alpha) {
+      v = VideoData::CreateAndCopyData(mInfo,
+                                       mImageContainer,
+                                       aSample->mOffset,
+                                       aSample->mTime,
+                                       aSample->mDuration,
+                                       b,
+                                       aSample->mKeyframe,
+                                       aSample->mTimecode,
+                                       mInfo.ScaledImageRect(img->d_w,
+                                                             img->d_h));
+    } else {
+      VideoData::YCbCrBuffer::Plane alpha_plane;
+      alpha_plane.mData = img_alpha->planes[0];
+      alpha_plane.mStride = img_alpha->stride[0];
+      alpha_plane.mHeight = img_alpha->d_h;
+      alpha_plane.mWidth = img_alpha->d_w;
+      alpha_plane.mOffset = alpha_plane.mSkip = 0;
+      v = VideoData::CreateAndCopyData(mInfo,
+                                       mImageContainer,
+                                       aSample->mOffset,
+                                       aSample->mTime,
+                                       aSample->mDuration,
+                                       b,
+                                       alpha_plane,
+                                       aSample->mKeyframe,
+                                       aSample->mTimecode,
+                                       mInfo.ScaledImageRect(img->d_w,
+                                                             img->d_h));
+
+    }
 
     if (!v) {
       LOG("Image allocation error source %ldx%ld display %ldx%ld picture %ldx%ld",
@@ -234,6 +291,32 @@ VPXDecoder::Drain()
   mTaskQueue->Dispatch(NewRunnableMethod(this, &VPXDecoder::ProcessDrain));
 }
 
+MediaResult
+VPXDecoder::DecodeAlpha(vpx_image_t** aImgAlpha,
+                        MediaRawData* aSample)
+{
+  vpx_codec_err_t r = vpx_codec_decode(&mVPXAlpha,
+                                       aSample->AlphaData(),
+                                       aSample->AlphaSize(),
+                                       nullptr,
+                                       0);
+  if (r) {
+    LOG("VPX decode alpha error: %s", vpx_codec_err_to_string(r));
+    return MediaResult(
+      NS_ERROR_DOM_MEDIA_DECODE_ERR,
+      RESULT_DETAIL("VPX decode alpha error: %s", vpx_codec_err_to_string(r)));
+  }
+
+  vpx_codec_iter_t iter = nullptr;
+
+  *aImgAlpha = vpx_codec_get_frame(&mVPXAlpha, &iter);
+  NS_ASSERTION((*aImgAlpha)->fmt == VPX_IMG_FMT_I420 ||
+               (*aImgAlpha)->fmt == VPX_IMG_FMT_I444,
+               "WebM image format not I420 or I444");
+
+  return NS_OK;
+}
+
 /* static */
 bool
 VPXDecoder::IsVPX(const nsACString& aMimeType, uint8_t aCodecMask)
diff --git a/dom/media/platforms/agnostic/VPXDecoder.h b/dom/media/platforms/agnostic/VPXDecoder.h
index f7e63e3a92..25d9bc1d33 100644
--- a/dom/media/platforms/agnostic/VPXDecoder.h
+++ b/dom/media/platforms/agnostic/VPXDecoder.h
@@ -57,6 +57,8 @@ private:
   void ProcessDecode(MediaRawData* aSample);
   MediaResult DoDecode(MediaRawData* aSample);
   void ProcessDrain();
+  MediaResult DecodeAlpha(vpx_image_t** aImgAlpha,
+                          MediaRawData* aSample);
 
   const RefPtr<ImageContainer> mImageContainer;
   const RefPtr<TaskQueue> mTaskQueue;
@@ -66,6 +68,9 @@ private:
   // VPx decoder state
   vpx_codec_ctx_t mVPX;
 
+  // VPx alpha decoder state
+  vpx_codec_ctx_t mVPXAlpha;
+
   const VideoInfo& mInfo;
 
   const Codec mCodec;
diff --git a/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h b/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h
index c27f61aad9..95b156ff96 100644
--- a/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h
+++ b/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h
@@ -32,6 +32,13 @@ public:
   already_AddRefed<MediaDataDecoder>
   CreateVideoDecoder(const CreateDecoderParams& aParams) override
   {
+    // Temporary - forces use of VPXDecoder when alpha is present.
+    // Bug 1263836 will handle alpha scenario once implemented. It will shift
+    // the check for alpha to PDMFactory but not itself remove the need for a
+    // check.
+    if (aParams.VideoConfig().HasAlpha()) {
+      return nullptr;
+    }
     RefPtr<MediaDataDecoder> decoder =
       new FFmpegVideoDecoder<V>(mLib,
                                 aParams.mTaskQueue,
diff --git a/gfx/ycbcr/YCbCrUtils.cpp b/gfx/ycbcr/YCbCrUtils.cpp
index f5a4353e2f..0f9c2c8ebc 100644
--- a/gfx/ycbcr/YCbCrUtils.cpp
+++ b/gfx/ycbcr/YCbCrUtils.cpp
@@ -155,5 +155,26 @@ ConvertYCbCrToRGB(const layers::PlanarYCbCrData& aData,
   }
 }
 
+void
+ConvertYCbCrAToARGB(const uint8_t* aSrcY,
+                    const uint8_t* aSrcU,
+                    const uint8_t* aSrcV,
+                    const uint8_t* aSrcA,
+                    int aSrcStrideYA, int aSrcStrideUV,
+                    uint8_t* aDstARGB, int aDstStrideARGB,
+                    int aWidth, int aHeight) {
+
+  ConvertYCbCrAToARGB32(aSrcY,
+                        aSrcU,
+                        aSrcV,
+                        aSrcA,
+                        aDstARGB,
+                        aWidth,
+                        aHeight,
+                        aSrcStrideYA,
+                        aSrcStrideUV,
+                        aDstStrideARGB);
+}
+
 } // namespace gfx
 } // namespace mozilla
diff --git a/gfx/ycbcr/YCbCrUtils.h b/gfx/ycbcr/YCbCrUtils.h
index 1cd2e1c4fd..dcc7b5e9aa 100644
--- a/gfx/ycbcr/YCbCrUtils.h
+++ b/gfx/ycbcr/YCbCrUtils.h
@@ -24,6 +24,17 @@ ConvertYCbCrToRGB(const layers::PlanarYCbCrData& aData,
                   unsigned char* aDestBuffer,
                   int32_t aStride);
 
+// Currently this function only has support for I420 type.
+void
+ConvertYCbCrAToARGB(const uint8_t* aSrcY,
+                    const uint8_t* aSrcU,
+                    const uint8_t* aSrcV,
+                    const uint8_t* aSrcA,
+                    int aSrcStrideYA, int aSrcStrideUV,
+                    uint8_t* aDstARGB, int aDstStrideARGB,
+                    int aWidth, int aHeight);
+
+
 } // namespace gfx
 } // namespace mozilla
 
diff --git a/gfx/ycbcr/yuv_convert.cpp b/gfx/ycbcr/yuv_convert.cpp
index d3a8c53312..89f6dcfa06 100644
--- a/gfx/ycbcr/yuv_convert.cpp
+++ b/gfx/ycbcr/yuv_convert.cpp
@@ -550,6 +550,27 @@ void ScaleYCbCrToRGB32_deprecated(const uint8_t* y_buf,
   if (has_mmx)
     EMMS();
 }
+void ConvertYCbCrAToARGB32(const uint8_t* y_buf,
+                           const uint8_t* u_buf,
+                           const uint8_t* v_buf,
+                           const uint8_t* a_buf,
+                           uint8_t* argb_buf,
+                           int pic_width,
+                           int pic_height,
+                           int ya_pitch,
+                           int uv_pitch,
+                           int argb_pitch) {
+
+  // The downstream graphics stack expects an attenuated input, hence why the
+  // attenuation parameter is set.
+  DebugOnly<int> err = libyuv::I420AlphaToARGB(y_buf, ya_pitch,
+                                               u_buf, uv_pitch,
+                                               v_buf, uv_pitch,
+                                               a_buf, ya_pitch,
+                                               argb_buf, argb_pitch,
+                                               pic_width, pic_height, 1);
+  MOZ_ASSERT(!err);
+}
 
 } // namespace gfx
 } // namespace mozilla
diff --git a/gfx/ycbcr/yuv_convert.h b/gfx/ycbcr/yuv_convert.h
index 108e14b679..2e85ada0a3 100644
--- a/gfx/ycbcr/yuv_convert.h
+++ b/gfx/ycbcr/yuv_convert.h
@@ -106,6 +106,18 @@ void ScaleYCbCrToRGB32_deprecated(const uint8_t* yplane,
                                   Rotate view_rotate,
                                   ScaleFilter filter);
 
+void ConvertYCbCrAToARGB32(const uint8_t* yplane,
+                           const uint8_t* uplane,
+                           const uint8_t* vplane,
+                           const uint8_t* aplane,
+                           uint8_t* argbframe,
+                           int pic_width,
+                           int pic_height,
+                           int yastride,
+                           int uvstride,
+                           int argbstride);
+
+
 } // namespace gfx
 } // namespace mozilla