diff options
| -rw-r--r-- | dom/media/MediaData.cpp | 139 | ||||
| -rw-r--r-- | dom/media/MediaData.h | 11 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/VPXDecoder.cpp | 153 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/VPXDecoder.h | 5 | ||||
| -rw-r--r-- | dom/media/platforms/ffmpeg/FFmpegDecoderModule.h | 7 | ||||
| -rw-r--r-- | gfx/ycbcr/YCbCrUtils.cpp | 21 | ||||
| -rw-r--r-- | gfx/ycbcr/YCbCrUtils.h | 11 | ||||
| -rw-r--r-- | gfx/ycbcr/yuv_convert.cpp | 21 | ||||
| -rw-r--r-- | gfx/ycbcr/yuv_convert.h | 12 |
9 files changed, 315 insertions, 65 deletions
diff --git a/dom/media/MediaData.cpp b/dom/media/MediaData.cpp index c113b62fe6..9f1205a0e1 100644 --- a/dom/media/MediaData.cpp +++ b/dom/media/MediaData.cpp @@ -7,6 +7,8 @@ #include "MediaInfo.h" #include "VideoUtils.h" #include "ImageContainer.h" +#include "mozilla/layers/SharedRGBImage.h" +#include "YCbCrUtils.h" #include <stdint.h> @@ -89,6 +91,45 @@ ValidatePlane(const VideoData::YCbCrBuffer::Plane& aPlane) aPlane.mStride > 0 && aPlane.mWidth <= aPlane.mStride; } +static bool ValidateBufferAndPicture(const VideoData::YCbCrBuffer& aBuffer, + const IntRect& aPicture) +{ + // The following situation should never happen unless there is a bug + // in the decoder + if (aBuffer.mPlanes[1].mWidth != aBuffer.mPlanes[2].mWidth || + aBuffer.mPlanes[1].mHeight != aBuffer.mPlanes[2].mHeight) { + NS_ERROR("C planes with different sizes"); + return false; + } + + // The following situations could be triggered by invalid input + if (aPicture.width <= 0 || aPicture.height <= 0) { + // In debug mode, makes the error more noticeable + MOZ_ASSERT(false, "Empty picture rect"); + return false; + } + if (!ValidatePlane(aBuffer.mPlanes[0]) || + !ValidatePlane(aBuffer.mPlanes[1]) || + !ValidatePlane(aBuffer.mPlanes[2])) { + NS_WARNING("Invalid plane size"); + return false; + } + + // Ensure the picture size specified in the headers can be extracted out of + // the frame we've been supplied without indexing out of bounds. + CheckedUint32 xLimit = aPicture.x + CheckedUint32(aPicture.width); + CheckedUint32 yLimit = aPicture.y + CheckedUint32(aPicture.height); + if (!xLimit.isValid() || xLimit.value() > aBuffer.mPlanes[0].mStride || + !yLimit.isValid() || yLimit.value() > aBuffer.mPlanes[0].mHeight) + { + // The specified picture dimensions can't be contained inside the video + // frame, we'll stomp memory if we try to copy it. Fail. + NS_WARNING("Overflowing picture rect"); + return false; + } + return true; +} + VideoData::VideoData(int64_t aOffset, int64_t aTime, int64_t aDuration, @@ -242,36 +283,7 @@ VideoData::CreateAndCopyData(const VideoInfo& aInfo, return v.forget(); } - // The following situation should never happen unless there is a bug - // in the decoder - if (aBuffer.mPlanes[1].mWidth != aBuffer.mPlanes[2].mWidth || - aBuffer.mPlanes[1].mHeight != aBuffer.mPlanes[2].mHeight) { - NS_ERROR("C planes with different sizes"); - return nullptr; - } - - // The following situations could be triggered by invalid input - if (aPicture.width <= 0 || aPicture.height <= 0) { - // In debug mode, makes the error more noticeable - MOZ_ASSERT(false, "Empty picture rect"); - return nullptr; - } - if (!ValidatePlane(aBuffer.mPlanes[0]) || !ValidatePlane(aBuffer.mPlanes[1]) || - !ValidatePlane(aBuffer.mPlanes[2])) { - NS_WARNING("Invalid plane size"); - return nullptr; - } - - // Ensure the picture size specified in the headers can be extracted out of - // the frame we've been supplied without indexing out of bounds. - CheckedUint32 xLimit = aPicture.x + CheckedUint32(aPicture.width); - CheckedUint32 yLimit = aPicture.y + CheckedUint32(aPicture.height); - if (!xLimit.isValid() || xLimit.value() > aBuffer.mPlanes[0].mStride || - !yLimit.isValid() || yLimit.value() > aBuffer.mPlanes[0].mHeight) - { - // The specified picture dimensions can't be contained inside the video - // frame, we'll stomp memory if we try to copy it. Fail. - NS_WARNING("Overflowing picture rect"); + if (!ValidateBufferAndPicture(aBuffer, aPicture)) { return nullptr; } @@ -307,6 +319,73 @@ VideoData::CreateAndCopyData(const VideoInfo& aInfo, /* static */ already_AddRefed<VideoData> +VideoData::CreateAndCopyData(const VideoInfo& aInfo, + ImageContainer* aContainer, + int64_t aOffset, + int64_t aTime, + int64_t aDuration, + const YCbCrBuffer& aBuffer, + const YCbCrBuffer::Plane &aAlphaPlane, + bool aKeyframe, + int64_t aTimecode, + const IntRect& aPicture) +{ + if (!aContainer) { + // Create a dummy VideoData with no image. This gives us something to + // send to media streams if necessary. + RefPtr<VideoData> v(new VideoData(aOffset, + aTime, + aDuration, + aKeyframe, + aTimecode, + aInfo.mDisplay, + 0)); + return v.forget(); + } + + if (!ValidateBufferAndPicture(aBuffer, aPicture)) { + return nullptr; + } + + RefPtr<VideoData> v(new VideoData(aOffset, + aTime, + aDuration, + aKeyframe, + aTimecode, + aInfo.mDisplay, + 0)); + + // Convert from YUVA to BGRA format on the software side. + RefPtr<layers::SharedRGBImage> videoImage = + aContainer->CreateSharedRGBImage(); + v->mImage = videoImage; + + if (!v->mImage) { + return nullptr; + } + if (!videoImage->Allocate(IntSize(aBuffer.mPlanes[0].mWidth, + aBuffer.mPlanes[0].mHeight), + SurfaceFormat::B8G8R8A8)) { + return nullptr; + } + uint8_t* argb_buffer = videoImage->GetBuffer(); + IntSize size = videoImage->GetSize(); + + // The naming convention for libyuv and associated utils is word-order. + // The naming convention in the gfx stack is byte-order. + ConvertYCbCrAToARGB(aBuffer.mPlanes[0].mData, + aBuffer.mPlanes[1].mData, + aBuffer.mPlanes[2].mData, + aAlphaPlane.mData, + aBuffer.mPlanes[0].mStride, aBuffer.mPlanes[1].mStride, + argb_buffer, size.width * 4, + size.width, size.height); + + return v.forget(); +} + +/* static */ +already_AddRefed<VideoData> VideoData::CreateFromImage(const VideoInfo& aInfo, int64_t aOffset, int64_t aTime, diff --git a/dom/media/MediaData.h b/dom/media/MediaData.h index 7e93541baa..02a7162f42 100644 --- a/dom/media/MediaData.h +++ b/dom/media/MediaData.h @@ -474,6 +474,17 @@ public: int64_t aTimecode, const IntRect& aPicture); + static already_AddRefed<VideoData> CreateAndCopyData(const VideoInfo& aInfo, + ImageContainer* aContainer, + int64_t aOffset, + int64_t aTime, + int64_t aDuration, + const YCbCrBuffer &aBuffer, + const YCbCrBuffer::Plane &aAlphaPlane, + bool aKeyframe, + int64_t aTimecode, + const IntRect& aPicture); + static already_AddRefed<VideoData> CreateAndCopyIntoTextureClient(const VideoInfo& aInfo, int64_t aOffset, int64_t aTime, diff --git a/dom/media/platforms/agnostic/VPXDecoder.cpp b/dom/media/platforms/agnostic/VPXDecoder.cpp index 007ead0c7d..00cf7d85cc 100644 --- a/dom/media/platforms/agnostic/VPXDecoder.cpp +++ b/dom/media/platforms/agnostic/VPXDecoder.cpp @@ -35,6 +35,38 @@ static VPXDecoder::Codec MimeTypeToCodec(const nsACString& aMimeType) return VPXDecoder::Codec::Unknown; } +static nsresult +InitContext(vpx_codec_ctx_t* aCtx, + const VideoInfo& aInfo, + const VPXDecoder::Codec aCodec) +{ + int decode_threads = 2; + + vpx_codec_iface_t* dx = nullptr; + if (aCodec == VPXDecoder::Codec::VP8) { + dx = vpx_codec_vp8_dx(); + } + else if (aCodec == VPXDecoder::Codec::VP9) { + dx = vpx_codec_vp9_dx(); + if (aInfo.mDisplay.width >= 2048) { + decode_threads = 8; + } + else if (aInfo.mDisplay.width >= 1024) { + decode_threads = 4; + } + } + decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors()); + + vpx_codec_dec_cfg_t config; + config.threads = decode_threads; + config.w = config.h = 0; // set after decode + + if (!dx || vpx_codec_dec_init(aCtx, dx, &config, 0)) { + return NS_ERROR_FAILURE; + } + return NS_OK; +} + VPXDecoder::VPXDecoder(const CreateDecoderParams& aParams) : mImageContainer(aParams.mImageContainer) , mTaskQueue(aParams.mTaskQueue) @@ -45,6 +77,7 @@ VPXDecoder::VPXDecoder(const CreateDecoderParams& aParams) { MOZ_COUNT_CTOR(VPXDecoder); PodZero(&mVPX); + PodZero(&mVPXAlpha); } VPXDecoder::~VPXDecoder() @@ -56,34 +89,24 @@ void VPXDecoder::Shutdown() { vpx_codec_destroy(&mVPX); + vpx_codec_destroy(&mVPXAlpha); } RefPtr<MediaDataDecoder::InitPromise> VPXDecoder::Init() { - int decode_threads = 2; - - vpx_codec_iface_t* dx = nullptr; - if (mCodec == Codec::VP8) { - dx = vpx_codec_vp8_dx(); - } else if (mCodec == Codec::VP9) { - dx = vpx_codec_vp9_dx(); - if (mInfo.mDisplay.width >= 2048) { - decode_threads = 8; - } else if (mInfo.mDisplay.width >= 1024) { - decode_threads = 4; - } + if (NS_FAILED(InitContext(&mVPX, mInfo, mCodec))) { + return VPXDecoder::InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__); } - decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors()); - - vpx_codec_dec_cfg_t config; - config.threads = decode_threads; - config.w = config.h = 0; // set after decode - - if (!dx || vpx_codec_dec_init(&mVPX, dx, &config, 0)) { - return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + if (mInfo.HasAlpha()) { + if (NS_FAILED(InitContext(&mVPXAlpha, mInfo, mCodec))) { + return VPXDecoder::InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__); + } } - return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); + return VPXDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, + __func__); } void @@ -115,14 +138,27 @@ VPXDecoder::DoDecode(MediaRawData* aSample) RESULT_DETAIL("VPX error: %s", vpx_codec_err_to_string(r))); } - vpx_codec_iter_t iter = nullptr; - vpx_image_t *img; + vpx_codec_iter_t iter = nullptr; + vpx_image_t *img; + vpx_image_t *img_alpha = nullptr; + bool alpha_decoded = false; while ((img = vpx_codec_get_frame(&mVPX, &iter))) { NS_ASSERTION(img->fmt == VPX_IMG_FMT_I420 || img->fmt == VPX_IMG_FMT_I444, "WebM image format not I420 or I444"); - + NS_ASSERTION(!alpha_decoded, + "Multiple frames per packet that contains alpha"); + + if (aSample->AlphaSize() > 0) { + if(!alpha_decoded){ + MediaResult rv = DecodeAlpha(&img_alpha, aSample); + if (NS_FAILED(rv)) { + return(rv); + } + alpha_decoded = true; + } + } // Chroma shifts are rounded down as per the decoding examples in the SDK VideoData::YCbCrBuffer b; b.mPlanes[0].mData = img->planes[0]; @@ -174,17 +210,38 @@ VPXDecoder::DoDecode(MediaRawData* aSample) }(); // TODO: need a newer libvpx to support full color range - RefPtr<VideoData> v = - VideoData::CreateAndCopyData(mInfo, - mImageContainer, - aSample->mOffset, - aSample->mTime, - aSample->mDuration, - b, - aSample->mKeyframe, - aSample->mTimecode, - mInfo.ScaledImageRect(img->d_w, - img->d_h)); + RefPtr<VideoData> v; + if (!img_alpha) { + v = VideoData::CreateAndCopyData(mInfo, + mImageContainer, + aSample->mOffset, + aSample->mTime, + aSample->mDuration, + b, + aSample->mKeyframe, + aSample->mTimecode, + mInfo.ScaledImageRect(img->d_w, + img->d_h)); + } else { + VideoData::YCbCrBuffer::Plane alpha_plane; + alpha_plane.mData = img_alpha->planes[0]; + alpha_plane.mStride = img_alpha->stride[0]; + alpha_plane.mHeight = img_alpha->d_h; + alpha_plane.mWidth = img_alpha->d_w; + alpha_plane.mOffset = alpha_plane.mSkip = 0; + v = VideoData::CreateAndCopyData(mInfo, + mImageContainer, + aSample->mOffset, + aSample->mTime, + aSample->mDuration, + b, + alpha_plane, + aSample->mKeyframe, + aSample->mTimecode, + mInfo.ScaledImageRect(img->d_w, + img->d_h)); + + } if (!v) { LOG("Image allocation error source %ldx%ld display %ldx%ld picture %ldx%ld", @@ -234,6 +291,32 @@ VPXDecoder::Drain() mTaskQueue->Dispatch(NewRunnableMethod(this, &VPXDecoder::ProcessDrain)); } +MediaResult +VPXDecoder::DecodeAlpha(vpx_image_t** aImgAlpha, + MediaRawData* aSample) +{ + vpx_codec_err_t r = vpx_codec_decode(&mVPXAlpha, + aSample->AlphaData(), + aSample->AlphaSize(), + nullptr, + 0); + if (r) { + LOG("VPX decode alpha error: %s", vpx_codec_err_to_string(r)); + return MediaResult( + NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("VPX decode alpha error: %s", vpx_codec_err_to_string(r))); + } + + vpx_codec_iter_t iter = nullptr; + + *aImgAlpha = vpx_codec_get_frame(&mVPXAlpha, &iter); + NS_ASSERTION((*aImgAlpha)->fmt == VPX_IMG_FMT_I420 || + (*aImgAlpha)->fmt == VPX_IMG_FMT_I444, + "WebM image format not I420 or I444"); + + return NS_OK; +} + /* static */ bool VPXDecoder::IsVPX(const nsACString& aMimeType, uint8_t aCodecMask) diff --git a/dom/media/platforms/agnostic/VPXDecoder.h b/dom/media/platforms/agnostic/VPXDecoder.h index f7e63e3a92..25d9bc1d33 100644 --- a/dom/media/platforms/agnostic/VPXDecoder.h +++ b/dom/media/platforms/agnostic/VPXDecoder.h @@ -57,6 +57,8 @@ private: void ProcessDecode(MediaRawData* aSample); MediaResult DoDecode(MediaRawData* aSample); void ProcessDrain(); + MediaResult DecodeAlpha(vpx_image_t** aImgAlpha, + MediaRawData* aSample); const RefPtr<ImageContainer> mImageContainer; const RefPtr<TaskQueue> mTaskQueue; @@ -66,6 +68,9 @@ private: // VPx decoder state vpx_codec_ctx_t mVPX; + // VPx alpha decoder state + vpx_codec_ctx_t mVPXAlpha; + const VideoInfo& mInfo; const Codec mCodec; diff --git a/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h b/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h index c27f61aad9..95b156ff96 100644 --- a/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h +++ b/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h @@ -32,6 +32,13 @@ public: already_AddRefed<MediaDataDecoder> CreateVideoDecoder(const CreateDecoderParams& aParams) override { + // Temporary - forces use of VPXDecoder when alpha is present. + // Bug 1263836 will handle alpha scenario once implemented. It will shift + // the check for alpha to PDMFactory but not itself remove the need for a + // check. + if (aParams.VideoConfig().HasAlpha()) { + return nullptr; + } RefPtr<MediaDataDecoder> decoder = new FFmpegVideoDecoder<V>(mLib, aParams.mTaskQueue, diff --git a/gfx/ycbcr/YCbCrUtils.cpp b/gfx/ycbcr/YCbCrUtils.cpp index f5a4353e2f..0f9c2c8ebc 100644 --- a/gfx/ycbcr/YCbCrUtils.cpp +++ b/gfx/ycbcr/YCbCrUtils.cpp @@ -155,5 +155,26 @@ ConvertYCbCrToRGB(const layers::PlanarYCbCrData& aData, } } +void +ConvertYCbCrAToARGB(const uint8_t* aSrcY, + const uint8_t* aSrcU, + const uint8_t* aSrcV, + const uint8_t* aSrcA, + int aSrcStrideYA, int aSrcStrideUV, + uint8_t* aDstARGB, int aDstStrideARGB, + int aWidth, int aHeight) { + + ConvertYCbCrAToARGB32(aSrcY, + aSrcU, + aSrcV, + aSrcA, + aDstARGB, + aWidth, + aHeight, + aSrcStrideYA, + aSrcStrideUV, + aDstStrideARGB); +} + } // namespace gfx } // namespace mozilla diff --git a/gfx/ycbcr/YCbCrUtils.h b/gfx/ycbcr/YCbCrUtils.h index 1cd2e1c4fd..dcc7b5e9aa 100644 --- a/gfx/ycbcr/YCbCrUtils.h +++ b/gfx/ycbcr/YCbCrUtils.h @@ -24,6 +24,17 @@ ConvertYCbCrToRGB(const layers::PlanarYCbCrData& aData, unsigned char* aDestBuffer, int32_t aStride); +// Currently this function only has support for I420 type. +void +ConvertYCbCrAToARGB(const uint8_t* aSrcY, + const uint8_t* aSrcU, + const uint8_t* aSrcV, + const uint8_t* aSrcA, + int aSrcStrideYA, int aSrcStrideUV, + uint8_t* aDstARGB, int aDstStrideARGB, + int aWidth, int aHeight); + + } // namespace gfx } // namespace mozilla diff --git a/gfx/ycbcr/yuv_convert.cpp b/gfx/ycbcr/yuv_convert.cpp index d3a8c53312..89f6dcfa06 100644 --- a/gfx/ycbcr/yuv_convert.cpp +++ b/gfx/ycbcr/yuv_convert.cpp @@ -550,6 +550,27 @@ void ScaleYCbCrToRGB32_deprecated(const uint8_t* y_buf, if (has_mmx) EMMS(); } +void ConvertYCbCrAToARGB32(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* argb_buf, + int pic_width, + int pic_height, + int ya_pitch, + int uv_pitch, + int argb_pitch) { + + // The downstream graphics stack expects an attenuated input, hence why the + // attenuation parameter is set. + DebugOnly<int> err = libyuv::I420AlphaToARGB(y_buf, ya_pitch, + u_buf, uv_pitch, + v_buf, uv_pitch, + a_buf, ya_pitch, + argb_buf, argb_pitch, + pic_width, pic_height, 1); + MOZ_ASSERT(!err); +} } // namespace gfx } // namespace mozilla diff --git a/gfx/ycbcr/yuv_convert.h b/gfx/ycbcr/yuv_convert.h index 108e14b679..2e85ada0a3 100644 --- a/gfx/ycbcr/yuv_convert.h +++ b/gfx/ycbcr/yuv_convert.h @@ -106,6 +106,18 @@ void ScaleYCbCrToRGB32_deprecated(const uint8_t* yplane, Rotate view_rotate, ScaleFilter filter); +void ConvertYCbCrAToARGB32(const uint8_t* yplane, + const uint8_t* uplane, + const uint8_t* vplane, + const uint8_t* aplane, + uint8_t* argbframe, + int pic_width, + int pic_height, + int yastride, + int uvstride, + int argbstride); + + } // namespace gfx } // namespace mozilla |